diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index 8ad7055b3d6a4..1bd33187116fd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -28,8 +28,12 @@ #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" +#include "gc/shenandoah/shenandoahLock.hpp" #include "gc/shenandoah/shenandoahSimpleBitMap.hpp" +typedef ShenandoahLock ShenandoahRebuildLock; +typedef ShenandoahLocker ShenandoahRebuildLocker; + // Each ShenandoahHeapRegion is associated with a ShenandoahFreeSetPartitionId. enum class ShenandoahFreeSetPartitionId : uint8_t { Mutator, // Region is in the Mutator free set: available memory is available to mutators. @@ -234,7 +238,7 @@ class ShenandoahRegionPartitions { // Return available_in assuming caller does not hold the heap lock. In production builds, available is // returned without acquiring the lock. In debug builds, the global heap lock is acquired in order to // enforce a consistency assert. - inline size_t available_in_not_locked(ShenandoahFreeSetPartitionId which_partition) const { + inline size_t available_in_locked_for_rebuild(ShenandoahFreeSetPartitionId which_partition) const { assert (which_partition < NumPartitions, "selected free set must be valid"); shenandoah_assert_not_heaplocked(); #ifdef ASSERT @@ -316,6 +320,9 @@ class ShenandoahFreeSet : public CHeapObj { ShenandoahHeap* const _heap; ShenandoahRegionPartitions _partitions; + // This locks the rebuild process (in combination with the global heap lock) + ShenandoahRebuildLock _lock; + HeapWord* allocate_aligned_plab(size_t size, ShenandoahAllocRequest& req, ShenandoahHeapRegion* r); // Return the address of memory allocated, setting in_new_region to true iff the allocation is taken @@ -415,6 +422,11 @@ class ShenandoahFreeSet : public CHeapObj { ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions); + + ShenandoahRebuildLock* lock() { + return &_lock; + } + // Public because ShenandoahRegionPartitions assertions require access. inline size_t alloc_capacity(ShenandoahHeapRegion *r) const; inline size_t alloc_capacity(size_t idx) const; @@ -480,7 +492,10 @@ class ShenandoahFreeSet : public CHeapObj { // locked action can be seen by these unlocked functions. inline size_t capacity() const { return _partitions.capacity_of(ShenandoahFreeSetPartitionId::Mutator); } inline size_t used() const { return _partitions.used_by(ShenandoahFreeSetPartitionId::Mutator); } - inline size_t available() const { return _partitions.available_in_not_locked(ShenandoahFreeSetPartitionId::Mutator); } + inline size_t available() { + ShenandoahRebuildLocker locker(lock()); + return _partitions.available_in_locked_for_rebuild(ShenandoahFreeSetPartitionId::Mutator); + } HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 27ff45e67de19..ceb8a6a0e9864 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -1153,17 +1153,17 @@ ShenandoahGenerationalHeap::TransferResult ShenandoahFullGC::phase5_epilog() { } heap->collection_set()->clear(); - size_t young_cset_regions, old_cset_regions; - size_t first_old, last_old, num_old; - heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); - + size_t young_cset_regions, old_cset_regions, first_old, last_old, num_old; + ShenandoahFreeSet* free_set = heap->free_set(); + ShenandoahRebuildLocker rebuild_locker(free_set->lock()); + free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); // We also do not expand old generation size following Full GC because we have scrambled age populations and // no longer have objects separated by age into distinct regions. if (heap->mode()->is_generational()) { ShenandoahGenerationalFullGC::compute_balances(); } - heap->free_set()->finish_rebuild(young_cset_regions, old_cset_regions, num_old); + free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old); // Set mark incomplete because the marking bitmaps have been reset except pinned regions. heap->global_generation()->set_mark_incomplete(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 0c55613efcc23..184cee0de0659 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -745,14 +745,16 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { { ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset : ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); + ShenandoahFreeSet* free_set = heap->free_set(); ShenandoahHeapLocker locker(heap->lock()); + ShenandoahRebuildLocker rebuild_locker(free_set->lock()); size_t young_cset_regions, old_cset_regions; // We are preparing for evacuation. At this time, we ignore cset region tallies. size_t first_old, last_old, num_old; - heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); + free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); // Free set construction uses reserve quantities, because they are known to be valid here - heap->free_set()->finish_rebuild(young_cset_regions, old_cset_regions, num_old, true); + free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old, true); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index e6d41237c98a3..43c2d4ad47688 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -424,6 +424,7 @@ jint ShenandoahHeap::initialize() { // We are initializing free set. We ignore cset region tallies. size_t first_old, last_old, num_old; + ShenandoahRebuildLocker rebuild_locker(_free_set->lock()); _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); _free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old); } @@ -2575,6 +2576,7 @@ void ShenandoahHeap::rebuild_free_set(bool concurrent) { ShenandoahPhaseTimings::final_update_refs_rebuild_freeset : ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset); ShenandoahHeapLocker locker(lock()); + ShenandoahRebuildLocker rebuild_locker(_free_set->lock()); size_t young_cset_regions, old_cset_regions; size_t first_old_region, last_old_region, old_region_count; _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count); @@ -2600,8 +2602,8 @@ void ShenandoahHeap::rebuild_free_set(bool concurrent) { // Total old_available may have been expanded to hold anticipated promotions. We trigger if the fragmented available // memory represents more than 16 regions worth of data. Note that fragmentation may increase when we promote regular - // regions in place when many of these regular regions have an abundant amount of available memory within them. Fragmentation - // will decrease as promote-by-copy consumes the available memory within these partially consumed regions. + // regions in place when many of these regular regions have an abundant amount of available memory within them. + // Fragmentation will decrease as promote-by-copy consumes the available memory within these partially consumed regions. // // We consider old-gen to have excessive fragmentation if more than 12.5% of old-gen is free memory that resides // within partially consumed regions of memory. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index ac3107eb396ec..272588bb3c8d7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -509,13 +509,14 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset : ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); + ShenandoahFreeSet* free_set = heap->free_set(); ShenandoahHeapLocker locker(heap->lock()); - size_t cset_young_regions, cset_old_regions; - size_t first_old, last_old, num_old; - heap->free_set()->prepare_to_rebuild(cset_young_regions, cset_old_regions, first_old, last_old, num_old); - // This is just old-gen completion. No future budgeting required here. The only reason to rebuild the freeset here - // is in case there was any immediate old garbage identified. - heap->free_set()->finish_rebuild(cset_young_regions, cset_old_regions, num_old); + ShenandoahRebuildLocker rebuild_locker(free_set->lock()); + size_t cset_young_regions, cset_old_regions, first_old, last_old, num_old; + free_set->prepare_to_rebuild(cset_young_regions, cset_old_regions, first_old, last_old, num_old); + // This is completion of old-gen marking. We rebuild in order to reclaim immediate garbage and to + // prepare for subsequent mixed evacuations. + free_set->finish_rebuild(cset_young_regions, cset_old_regions, num_old); } }