8327097: GenShen: Align PLAB sizes down rather than up

src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp

@@ -29,6 +29,9 @@
 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
+#include "logging/log.hpp"
 #include "utilities/quickSort.hpp"
 
 uint ShenandoahOldHeuristics::NOT_FOUND = -1U;

src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp

@@ -217,8 +217,10 @@ void ShenandoahArguments::initialize_alignments() {
 }
 
 CollectedHeap* ShenandoahArguments::create_heap() {
-  if (strcmp(ShenandoahGCMode, "generational") == 0) {
+  if (strcmp(ShenandoahGCMode, "generational") != 0) {
+    // Not generational
+    return new ShenandoahHeap(new ShenandoahCollectorPolicy());
+  } else {
     return new ShenandoahGenerationalHeap(new ShenandoahCollectorPolicy());
   }
-  return new ShenandoahHeap(new ShenandoahCollectorPolicy());
 }

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp

@@ -459,10 +459,11 @@ HeapWord* ShenandoahFreeSet::allocate_old_with_affiliation(ShenandoahAffiliation
 
 void ShenandoahFreeSet::add_old_collector_free_region(ShenandoahHeapRegion* region) {
   shenandoah_assert_heaplocked();
+  size_t plab_min_size_in_bytes = ShenandoahGenerationalHeap::heap()->plab_min_size() * HeapWordSize;
   size_t idx = region->index();
   size_t capacity = alloc_capacity(region);
   assert(_free_sets.membership(idx) == NotFree, "Regions promoted in place should not be in any free set");
-  if (capacity >= PLAB::min_size() * HeapWordSize) {
+  if (capacity >= plab_min_size_in_bytes) {
     _free_sets.make_free(idx, OldCollector, capacity);
     _heap->old_generation()->augment_promoted_reserve(capacity);
   }
@@ -698,13 +699,13 @@ HeapWord* ShenandoahFreeSet::allocate_aligned_plab(size_t size, ShenandoahAllocR
   assert(_heap->mode()->is_generational(), "PLABs are only for generational mode");
   assert(r->is_old(), "All PLABs reside in old-gen");
   assert(!req.is_mutator_alloc(), "PLABs should not be allocated by mutators.");
-  assert(size % CardTable::card_size_in_words() == 0, "size must be multiple of card table size, was " SIZE_FORMAT, size);
+  assert(is_aligned(size, CardTable::card_size_in_words()), "Align by design");
 
   HeapWord* result = r->allocate_aligned(size, req, CardTable::card_size());
   assert(result != nullptr, "Allocation cannot fail");
   assert(r->top() <= r->end(), "Allocation cannot span end of region");
   assert(req.actual_size() == size, "Should not have needed to adjust size for PLAB.");
-  assert(((uintptr_t) result) % CardTable::card_size_in_words() == 0, "PLAB start must align with card boundary");
+  assert(is_aligned(result, CardTable::card_size_in_words()), "Align by design");
 
   return result;
 }
@@ -1066,7 +1067,7 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi
       assert(!region->is_cset(), "Shouldn't be adding cset regions to the free set");
       assert(_free_sets.in_free_set(idx, NotFree), "We are about to make region free; it should not be free already");
 
-      // Do not add regions that would almost surely fail allocation
+      // Do not add regions that would almost surely fail allocation.  Note that PLAB::min_size() is typically less than ShenandoahGenerationalHeap::plab_min_size()
       if (alloc_capacity(region) < PLAB::min_size() * HeapWordSize) continue;
 
       if (region->is_old()) {

src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp

@@ -72,9 +72,25 @@ ShenandoahGenerationalHeap* ShenandoahGenerationalHeap::heap() {
   return checked_cast<ShenandoahGenerationalHeap*>(heap);
 }
 
+size_t ShenandoahGenerationalHeap::calculate_min_plab() const {
+  return align_up(PLAB::min_size(), CardTable::card_size_in_words());
+}
+
+size_t ShenandoahGenerationalHeap::calculate_max_plab() const {
+  size_t MaxTLABSizeWords = ShenandoahHeapRegion::max_tlab_size_words();
+  return ((ShenandoahMaxEvacLABRatio > 0)?
+          align_down(MIN2(MaxTLABSizeWords, PLAB::min_size() * ShenandoahMaxEvacLABRatio), CardTable::card_size_in_words()):
+          align_down(MaxTLABSizeWords, CardTable::card_size_in_words()));
+}
+
 ShenandoahGenerationalHeap::ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy) :
   ShenandoahHeap(policy),
-  _regulator_thread(nullptr) { }
+  _min_plab_size(calculate_min_plab()),
+  _max_plab_size(calculate_max_plab()),
+  _regulator_thread(nullptr) {
+  assert(is_aligned(_min_plab_size, CardTable::card_size_in_words()), "min_plab_size must be aligned");
+  assert(is_aligned(_max_plab_size, CardTable::card_size_in_words()), "max_plab_size must be aligned");
+}
 
 void ShenandoahGenerationalHeap::print_init_logger() const {
   ShenandoahGenerationalInitLogger logger;
@@ -265,4 +281,4 @@ void ShenandoahGenerationalHeap::TransferResult::print_on(const char* when, outp
                      when,
                      success? "successfully transferred": "failed to transfer", region_count, region_destination,
                      PROPERFMTARGS(old_available), PROPERFMTARGS(young_available));
-}
+}

src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp

@@ -31,11 +31,22 @@ class ShenandoahRegulatorThread;
 class ShenandoahGenerationalControlThread;
 
 class ShenandoahGenerationalHeap : public ShenandoahHeap {
+private:
+  const size_t _min_plab_size;
+  const size_t _max_plab_size;
+
+  size_t calculate_min_plab() const;
+  size_t calculate_max_plab() const;
+
 public:
   explicit ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy);
 
+
   static ShenandoahGenerationalHeap* heap();
 
+  inline size_t plab_min_size() const { return _min_plab_size; }
+  inline size_t plab_max_size() const { return _max_plab_size; }
+
   void print_init_logger() const override;
 
   // ---------- Serviceability

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -51,6 +51,7 @@
 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
 #include "gc/shenandoah/shenandoahControlThread.hpp"
 #include "gc/shenandoah/shenandoahFreeSet.hpp"
+#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
@@ -298,7 +299,7 @@ jint ShenandoahHeap::initialize() {
   _bitmap_region_special = bitmap.special();
 
   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
-                              align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
+    align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
   if (!_bitmap_region_special) {
     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
@@ -1024,26 +1025,27 @@ HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size)
 // Establish a new PLAB and allocate size HeapWords within it.
 HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, bool is_promotion) {
   // New object should fit the PLAB size
-  size_t min_size = MAX2(size, PLAB::min_size());
 
-  // Figure out size of new PLAB, looking back at heuristics. Expand aggressively.
+  assert(mode()->is_generational(), "PLABs only relevant to generational GC");
+  ShenandoahGenerationalHeap* generational_heap = (ShenandoahGenerationalHeap*) this;
+  const size_t plab_min_size = generational_heap->plab_min_size();
+  const size_t min_size = (size > plab_min_size)? align_up(size, CardTable::card_size_in_words()): plab_min_size;
+
+  // Figure out size of new PLAB, looking back at heuristics. Expand aggressively.  PLABs must align on size
+  // of card table in order to avoid the need for synchronization when registering newly allocated objects within
+  // the card table.
   size_t cur_size = ShenandoahThreadLocalData::plab_size(thread);
   if (cur_size == 0) {
-    cur_size = PLAB::min_size();
-  }
-  size_t future_size = cur_size * 2;
-  // Limit growth of PLABs to ShenandoahMaxEvacLABRatio * the minimum size.  This enables more equitable distribution of
-  // available evacuation buidget between the many threads that are coordinating in the evacuation effort.
-  if (ShenandoahMaxEvacLABRatio > 0) {
-    future_size = MIN2(future_size, PLAB::min_size() * ShenandoahMaxEvacLABRatio);
+    cur_size = plab_min_size;
   }
-  future_size = MIN2(future_size, PLAB::max_size());
-  future_size = MAX2(future_size, PLAB::min_size());
 
-  size_t unalignment = future_size % CardTable::card_size_in_words();
-  if (unalignment != 0) {
-    future_size = future_size - unalignment + CardTable::card_size_in_words();
-  }
+  // Limit growth of PLABs to the smaller of ShenandoahMaxEvacLABRatio * the minimum size and ShenandoahHumongousThreshold.
+  // This minimum value is represented by generational_heap->plab_max_size().  Enforcing this limit enables more equitable
+  // distribution of available evacuation budget between the many threads that are coordinating in the evacuation effort.
+  size_t future_size = MIN2(cur_size * 2, generational_heap->plab_max_size());
+  assert(is_aligned(future_size, CardTable::card_size_in_words()), "Align by design, future_size: " SIZE_FORMAT
+         ", alignment: " SIZE_FORMAT ", cur_size: " SIZE_FORMAT ", max: " SIZE_FORMAT,
+         future_size, (size_t) CardTable::card_size_in_words(), cur_size, generational_heap->plab_max_size());
 
   // Record new heuristic value even if we take any shortcut. This captures
   // the case when moderately-sized objects always take a shortcut. At some point,
@@ -1058,7 +1060,7 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b
 
   // Retire current PLAB, and allocate a new one.
   PLAB* plab = ShenandoahThreadLocalData::plab(thread);
-  if (plab->words_remaining() < PLAB::min_size()) {
+  if (plab->words_remaining() < plab_min_size) {
     // Retire current PLAB, and allocate a new one.
     // CAUTION: retire_plab may register the remnant filler object with the remembered set scanner without a lock.  This
     // is safe iff it is assured that each PLAB is a whole-number multiple of card-mark memory size and each PLAB is
@@ -1070,7 +1072,7 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b
     // less than the remaining evacuation need.  It also adjusts plab_preallocated and expend_promoted if appropriate.
     HeapWord* plab_buf = allocate_new_plab(min_size, cur_size, &actual_size);
     if (plab_buf == nullptr) {
-      if (min_size == PLAB::min_size()) {
+      if (min_size == plab_min_size) {
         // Disable plab promotions for this thread because we cannot even allocate a plab of minimal size.  This allows us
         // to fail faster on subsequent promotion attempts.
         ShenandoahThreadLocalData::disable_plab_promotions(thread);
@@ -1079,7 +1081,7 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b
     } else {
       ShenandoahThreadLocalData::enable_plab_retries(thread);
     }
-    assert (size <= actual_size, "allocation should fit");
+    // Since the allocated PLAB may have been down-sized for alignment, plab->allocate(size) below may still fail.
     if (ZeroTLAB) {
       // ..and clear it.
       Copy::zero_to_words(plab_buf, actual_size);
@@ -1093,6 +1095,7 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b
       Copy::fill_to_words(plab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 #endif // ASSERT
     }
+    assert(is_aligned(actual_size, CardTable::card_size_in_words()), "Align by design");
     plab->set_buf(plab_buf, actual_size);
     if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) {
       return nullptr;
@@ -1129,15 +1132,17 @@ void ShenandoahHeap::retire_plab(PLAB* plab, Thread* thread) {
   if (not_promoted > 0) {
     old_generation()->unexpend_promoted(not_promoted);
   }
-  size_t waste = plab->waste();
-  HeapWord* top = plab->top();
+  const size_t original_waste = plab->waste();
+  HeapWord* const top = plab->top();
+
+  // plab->retire() overwrites unused memory between plab->top() and plab->hard_end() with a dummy object to make memory parsable.
+  // It adds the size of this unused memory, in words, to plab->waste().
   plab->retire();
-  if (top != nullptr && plab->waste() > waste && is_in_old(top)) {
-    // If retiring the plab created a filler object, then we
-    // need to register it with our card scanner so it can
+  if (top != nullptr && plab->waste() > original_waste && is_in_old(top)) {
+    // If retiring the plab created a filler object, then we need to register it with our card scanner so it can
     // safely walk the region backing the plab.
     log_debug(gc)("retire_plab() is registering remnant of size " SIZE_FORMAT " at " PTR_FORMAT,
-                  plab->waste() - waste, p2i(top));
+                  plab->waste() - original_waste, p2i(top));
     card_scan()->register_object_without_lock(top);
   }
 }
@@ -1195,14 +1200,11 @@ HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
   return res;
 }
 
-HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size,
-                                            size_t word_size,
-                                            size_t* actual_size) {
-  // Align requested sizes to card sized multiples
-  size_t words_in_card = CardTable::card_size_in_words();
-  size_t align_mask = ~(words_in_card - 1);
-  min_size = (min_size + words_in_card - 1) & align_mask;
-  word_size = (word_size + words_in_card - 1) & align_mask;
+HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size, size_t word_size, size_t* actual_size) {
+  // Align requested sizes to card-sized multiples.  Align down so that we don't violate max size of TLAB.
+  assert(is_aligned(min_size, CardTable::card_size_in_words()), "Align by design");
+  assert(word_size >= min_size, "Requested PLAB is too small");
+
   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size);
   // Note that allocate_memory() sets a thread-local flag to prohibit further promotions by this thread
   // if we are at risk of infringing on the old-gen evacuation budget.
@@ -1212,6 +1214,7 @@ HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size,
   } else {
     *actual_size = 0;
   }
+  assert(is_aligned(res, CardTable::card_size_in_words()), "Align by design");
   return res;
 }
 
@@ -1723,6 +1726,8 @@ oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapReg
           break;
         }
         case OLD_GENERATION: {
+          assert(mode()->is_generational(), "OLD Generation only exists in generational mode");
+          ShenandoahGenerationalHeap* gen_heap = (ShenandoahGenerationalHeap*) this;
           PLAB* plab = ShenandoahThreadLocalData::plab(thread);
           if (plab != nullptr) {
             has_plab = true;
@@ -1732,13 +1737,13 @@ oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapReg
               ShenandoahThreadLocalData::plab_retries_enabled(thread)) {
             // PLAB allocation failed because we are bumping up against the limit on old evacuation reserve or because
             // the requested object does not fit within the current plab but the plab still has an "abundance" of memory,
-            // where abundance is defined as >= PLAB::min_size().  In the former case, we try resetting the desired
+            // where abundance is defined as >= ShenGenHeap::plab_min_size().  In the former case, we try resetting the desired
             // PLAB size and retry PLAB allocation to avoid cascading of shared memory allocations.
 
             // In this situation, PLAB memory is precious.  We'll try to preserve our existing PLAB by forcing
             // this particular allocation to be shared.
-            if (plab->words_remaining() < PLAB::min_size()) {
-              ShenandoahThreadLocalData::set_plab_size(thread, PLAB::min_size());
+            if (plab->words_remaining() < gen_heap->plab_min_size()) {
+              ShenandoahThreadLocalData::set_plab_size(thread, gen_heap->plab_min_size());
               copy = allocate_from_plab(thread, size, is_promotion);
               // If we still get nullptr, we'll try a shared allocation below.
               if (copy == nullptr) {

src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp

@@ -313,11 +313,11 @@ inline HeapWord* ShenandoahHeap::allocate_from_plab(Thread* thread, size_t size,
   }
   // if plab->word_size() <= 0, thread's plab not yet initialized for this pass, so allow_plab_promotions() is not trustworthy
   obj = plab->allocate(size);
-  if ((obj == nullptr) && (plab->words_remaining() < PLAB::min_size())) {
+  if ((obj == nullptr) && (plab->words_remaining() < ShenandoahGenerationalHeap::heap()->plab_min_size())) {
     // allocate_from_plab_slow will establish allow_plab_promotions(thread) for future invocations
     obj = allocate_from_plab_slow(thread, size, is_promotion);
   }
-  // if plab->words_remaining() >= PLAB::min_size(), just return nullptr so we can use a shared allocation
+  // if plab->words_remaining() >= ShenGenHeap::heap()->plab_min_size(), just return nullptr so we can use a shared allocation
   if (obj == nullptr) {
     return nullptr;
   }

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp

@@ -1064,7 +1064,7 @@ void ShenandoahHeapRegion::promote_in_place() {
     old_gen->increment_affiliated_region_count();
     old_gen->increase_used(region_used);
 
-    // add_old_collector_free_region() increases promoted_reserve() if available space exceeds PLAB::min_size()
+    // add_old_collector_free_region() increases promoted_reserve() if available space exceeds plab_min_size()
     heap->free_set()->add_old_collector_free_region(this);
   }
 }

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -351,8 +351,11 @@ class ShenandoahHeapRegion {
   inline void restore_top_before_promote();
   inline size_t garbage_before_padded_for_promote() const;
 
-  // Allocation (return nullptr if full)
-  inline HeapWord* allocate_aligned(size_t word_size, ShenandoahAllocRequest &req, size_t alignment_in_words);
+  // If next available memory is not aligned on address that is multiple of alignment, fill the empty space
+  // so that returned object is aligned on an address that is a multiple of alignment_in_bytes.  Requested
+  // size is in words.  It is assumed that this->is_old().  A pad object is allocated, filled, and registered
+  // if necessary to assure the new allocation is properly aligned.  Return nullptr if memory is not available.
+  inline HeapWord* allocate_aligned(size_t word_size, ShenandoahAllocRequest &req, size_t alignment_in_bytes);
 
   // Allocation (return nullptr if full)
   inline HeapWord* allocate(size_t word_size, ShenandoahAllocRequest req);