8345423: Shenandoah: Parallelize concurrent cleanup

Reviewed-by: ysr, kdnilsen, wkemper

Author: Xiaolong Peng

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -1046,7 +1046,10 @@ void ShenandoahConcurrentGC::op_strong_roots() {
 }
 
 void ShenandoahConcurrentGC::op_cleanup_early() {
-  ShenandoahHeap::heap()->free_set()->recycle_trash();
+  ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
+                              ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
+                              "cleanup early.");
+  ShenandoahHeap::heap()->recycle_trash();
 }
 
 void ShenandoahConcurrentGC::op_evacuate() {
@@ -1178,7 +1181,10 @@ void ShenandoahConcurrentGC::op_final_roots() {
 }
 
 void ShenandoahConcurrentGC::op_cleanup_complete() {
-  ShenandoahHeap::heap()->free_set()->recycle_trash();
+  ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
+                              ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup(),
+                              "cleanup complete.");
+  ShenandoahHeap::heap()->recycle_trash();
 }
 
 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp

@@ -729,7 +729,6 @@ void ShenandoahRegionPartitions::assert_bounds() {
 ShenandoahFreeSet::ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions) :
   _heap(heap),
   _partitions(max_regions, this),
-  _trash_regions(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, max_regions, mtGC)),
   _alloc_bias_weight(0)
 {
   clear_internal();
@@ -1002,7 +1001,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah
     return nullptr;
   }
   HeapWord* result = nullptr;
-  try_recycle_trashed(r);
+  r->try_recycle_under_lock();
   in_new_region = r->is_empty();
 
   if (in_new_region) {
@@ -1213,7 +1212,7 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) {
   // Initialize regions:
   for (idx_t i = beg; i <= end; i++) {
     ShenandoahHeapRegion* r = _heap->get_region(i);
-    try_recycle_trashed(r);
+    r->try_recycle_under_lock();
 
     assert(i == beg || _heap->get_region(i - 1)->index() + 1 == r->index(), "Should be contiguous");
     assert(r->is_empty(), "Should be empty");
@@ -1255,63 +1254,28 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) {
   return _heap->get_region(beg)->bottom();
 }
 
-void ShenandoahFreeSet::try_recycle_trashed(ShenandoahHeapRegion* r) {
-  if (r->is_trash()) {
-    r->recycle();
+class ShenandoahRecycleTrashedRegionClosure final : public ShenandoahHeapRegionClosure {
+public:
+  ShenandoahRecycleTrashedRegionClosure(): ShenandoahHeapRegionClosure() {}
+
+  void heap_region_do(ShenandoahHeapRegion* r) {
+    r->try_recycle();
   }
-}
+
+  bool is_thread_safe() {
+    return true;
+  }
+};
 
 void ShenandoahFreeSet::recycle_trash() {
-  // lock is not reentrable, check we don't have it
+  // lock is not non-reentrant, check we don't have it
   shenandoah_assert_not_heaplocked();
-  size_t count = 0;
-  for (size_t i = 0; i < _heap->num_regions(); i++) {
-    ShenandoahHeapRegion* r = _heap->get_region(i);
-    if (r->is_trash()) {
-      _trash_regions[count++] = r;
-    }
-  }
 
-  size_t total_batches = 0;
-  jlong batch_start_time = 0;
-  jlong recycle_trash_start_time = os::javaTimeNanos();    // This value will be treated as the initial batch_start_time
-  jlong batch_end_time = recycle_trash_start_time;
-  // Process as many batches as can be processed within 10 us.
-  static constexpr jlong deadline_ns = 10000;               // 10 us
-  size_t idx = 0;
-  jlong predicted_next_batch_end_time;
-  jlong batch_process_time_estimate = 0;
-  while (idx < count) {
-    if (idx > 0) {
-      os::naked_yield(); // Yield to allow allocators to take the lock, except on the first iteration
-    }
-    // Avoid another call to javaTimeNanos() if we already know time at which last batch ended
-    batch_start_time = batch_end_time;
-    const jlong deadline = batch_start_time + deadline_ns;
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
+  heap->assert_gc_workers(heap->workers()->active_workers());
 
-    ShenandoahHeapLocker locker(_heap->lock());
-    do {
-      // Measurements on typical 2024 hardware suggest it typically requires between 1400 and 2000 ns to process a batch of
-      // 32 regions, assuming low contention with other threads.  Sometimes this goes higher, when mutator threads
-      // are contending for CPU cores and/or the heap lock.  On this hardware with a 10 us deadline, we expect 3-6 batches
-      // to be processed between yields most of the time.
-      //
-      // Note that deadline is enforced since the end of previous batch.  In the case that yield() or acquisition of heap lock
-      // takes a "long time", we will have less time to process regions, but we will always process at least one batch between
-      // yields.  Yielding more frequently when there is heavy contention for the heap lock or for CPU cores is considered the
-      // right thing to do.
-      const size_t REGIONS_PER_BATCH = 32;
-      size_t max_idx = MIN2(count, idx + REGIONS_PER_BATCH);
-      while (idx < max_idx) {
-        try_recycle_trashed(_trash_regions[idx++]);
-      }
-      total_batches++;
-      batch_end_time = os::javaTimeNanos();
-      // Estimate includes historic combination of yield times and heap lock acquisition times.
-      batch_process_time_estimate = (batch_end_time - recycle_trash_start_time) / total_batches;
-      predicted_next_batch_end_time = batch_end_time + batch_process_time_estimate;
-    } while ((idx < count) && (predicted_next_batch_end_time < deadline));
-  }
+  ShenandoahRecycleTrashedRegionClosure closure;
+  heap->parallel_heap_region_iterate(&closure);
 }
 
 void ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) {

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp

@@ -286,7 +286,6 @@ class ShenandoahFreeSet : public CHeapObj<mtGC> {
 private:
   ShenandoahHeap* const _heap;
   ShenandoahRegionPartitions _partitions;
-  ShenandoahHeapRegion** _trash_regions;
 
   HeapWord* allocate_aligned_plab(size_t size, ShenandoahAllocRequest& req, ShenandoahHeapRegion* r);
 
@@ -352,7 +351,6 @@ class ShenandoahFreeSet : public CHeapObj<mtGC> {
   HeapWord* try_allocate_from_mutator(ShenandoahAllocRequest& req, bool& in_new_region);
 
   void clear_internal();
-  void try_recycle_trashed(ShenandoahHeapRegion *r);
 
   // Returns true iff this region is entirely available, either because it is empty() or because it has been found to represent
   // immediate trash and we'll be able to immediately recycle it.  Note that we cannot recycle immediate trash if

src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp

@@ -541,7 +541,7 @@ class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
   ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {}
   void heap_region_do(ShenandoahHeapRegion* r) {
     if (r->is_trash()) {
-      r->recycle();
+      r->try_recycle_under_lock();
     }
     if (r->is_cset()) {
       // Leave affiliation unchanged
@@ -990,7 +990,7 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
     // Recycle all trash regions
     if (r->is_trash()) {
       live = 0;
-      r->recycle();
+      r->try_recycle_under_lock();
     } else {
       if (r->is_old()) {
         ShenandoahGenerationalFullGC::account_for_region(r, _old_regions, _old_usage, _old_humongous_waste);

src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp

@@ -883,43 +883,45 @@ size_t ShenandoahGeneration::increment_affiliated_region_count() {
   // During full gc, multiple GC worker threads may change region affiliations without a lock.  No lock is enforced
   // on read and write of _affiliated_region_count.  At the end of full gc, a single thread overwrites the count with
   // a coherent value.
-  _affiliated_region_count++;
-  return _affiliated_region_count;
+  return Atomic::add(&_affiliated_region_count, (size_t) 1);
 }
 
 size_t ShenandoahGeneration::decrement_affiliated_region_count() {
   shenandoah_assert_heaplocked_or_safepoint();
   // During full gc, multiple GC worker threads may change region affiliations without a lock.  No lock is enforced
   // on read and write of _affiliated_region_count.  At the end of full gc, a single thread overwrites the count with
   // a coherent value.
-  _affiliated_region_count--;
+  auto affiliated_region_count = Atomic::sub(&_affiliated_region_count, (size_t) 1);
   assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
-         (_used + _humongous_waste <= _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()),
+         (used() + _humongous_waste <= affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()),
          "used + humongous cannot exceed regions");
-  return _affiliated_region_count;
+  return affiliated_region_count;
+}
+
+size_t ShenandoahGeneration::decrement_affiliated_region_count_without_lock() {
+  return Atomic::sub(&_affiliated_region_count, (size_t) 1);
 }
 
 size_t ShenandoahGeneration::increase_affiliated_region_count(size_t delta) {
   shenandoah_assert_heaplocked_or_safepoint();
-  _affiliated_region_count += delta;
-  return _affiliated_region_count;
+  return Atomic::add(&_affiliated_region_count, delta);
 }
 
 size_t ShenandoahGeneration::decrease_affiliated_region_count(size_t delta) {
   shenandoah_assert_heaplocked_or_safepoint();
-  assert(_affiliated_region_count >= delta, "Affiliated region count cannot be negative");
+  assert(Atomic::load(&_affiliated_region_count) >= delta, "Affiliated region count cannot be negative");
 
-  _affiliated_region_count -= delta;
+  auto const affiliated_region_count = Atomic::sub(&_affiliated_region_count, delta);
   assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
-         (_used + _humongous_waste <= _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()),
+         (_used + _humongous_waste <= affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()),
          "used + humongous cannot exceed regions");
-  return _affiliated_region_count;
+  return affiliated_region_count;
 }
 
 void ShenandoahGeneration::establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste) {
   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
-  _affiliated_region_count = num_regions;
-  _used = num_bytes;
+  Atomic::store(&_affiliated_region_count, num_regions);
+  Atomic::store(&_used, num_bytes);
   _humongous_waste = humongous_waste;
 }
 
@@ -948,21 +950,22 @@ void ShenandoahGeneration::decrease_used(size_t bytes) {
 }
 
 size_t ShenandoahGeneration::used_regions() const {
-  return _affiliated_region_count;
+  return Atomic::load(&_affiliated_region_count);
 }
 
 size_t ShenandoahGeneration::free_unaffiliated_regions() const {
   size_t result = max_capacity() / ShenandoahHeapRegion::region_size_bytes();
-  if (_affiliated_region_count > result) {
+  auto const used_regions = this->used_regions();
+  if (used_regions > result) {
     result = 0;
   } else {
-    result -= _affiliated_region_count;
+    result -= used_regions;
   }
   return result;
 }
 
 size_t ShenandoahGeneration::used_regions_size() const {
-  return _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes();
+  return used_regions() * ShenandoahHeapRegion::region_size_bytes();
 }
 
 size_t ShenandoahGeneration::available() const {
@@ -996,7 +999,7 @@ size_t ShenandoahGeneration::increase_capacity(size_t increment) {
 
   // This detects arithmetic wraparound on _used
   assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
-         (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() >= _used),
+         (used_regions_size() >= used()),
          "Affiliated regions must hold more than what is currently used");
   return _max_capacity;
 }
@@ -1020,12 +1023,12 @@ size_t ShenandoahGeneration::decrease_capacity(size_t decrement) {
 
   // This detects arithmetic wraparound on _used
   assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
-         (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() >= _used),
+         (used_regions_size() >= used()),
          "Affiliated regions must hold more than what is currently used");
   assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
          (_used <= _max_capacity), "Cannot use more than capacity");
   assert(ShenandoahHeap::heap()->is_full_gc_in_progress() ||
-         (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() <= _max_capacity),
+         (used_regions_size() <= _max_capacity),
          "Cannot use more than capacity");
   return _max_capacity;
 }

src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp

@@ -52,7 +52,7 @@ class ShenandoahGeneration : public CHeapObj<mtGC>, public ShenandoahSpaceInfo {
 
   ShenandoahReferenceProcessor* const _ref_processor;
 
-  size_t _affiliated_region_count;
+  volatile size_t _affiliated_region_count;
 
   // How much free memory is left in the last region of humongous objects.
   // This is _not_ included in used, but it _is_ deducted from available,
@@ -131,7 +131,7 @@ class ShenandoahGeneration : public CHeapObj<mtGC>, public ShenandoahSpaceInfo {
   virtual size_t used_regions() const;
   virtual size_t used_regions_size() const;
   virtual size_t free_unaffiliated_regions() const;
-  size_t used() const override { return _used; }
+  size_t used() const override { return Atomic::load(&_used); }
   size_t available() const override;
   size_t available_with_reserve() const;
   size_t used_including_humongous_waste() const {
@@ -219,6 +219,8 @@ class ShenandoahGeneration : public CHeapObj<mtGC>, public ShenandoahSpaceInfo {
 
   // Return the updated value of affiliated_region_count
   size_t decrement_affiliated_region_count();
+  // Same as decrement_affiliated_region_count, but w/o the need to hold heap lock before being called.
+  size_t decrement_affiliated_region_count_without_lock();
 
   // Return the updated value of affiliated_region_count
   size_t increase_affiliated_region_count(size_t delta);

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -605,10 +605,10 @@ class ShenandoahHeap : public CollectedHeap {
   // such an object as unreachable.
   inline bool is_in_old_during_young_collection(oop obj) const;
 
-  inline ShenandoahAffiliation region_affiliation(const ShenandoahHeapRegion* r);
+  inline ShenandoahAffiliation region_affiliation(const ShenandoahHeapRegion* r) const;
   inline void set_affiliation(ShenandoahHeapRegion* r, ShenandoahAffiliation new_affiliation);
 
-  inline ShenandoahAffiliation region_affiliation(size_t index);
+  inline ShenandoahAffiliation region_affiliation(size_t index) const;
 
   bool requires_barriers(stackChunkOop obj) const override;
 

src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp

@@ -371,7 +371,7 @@ inline bool ShenandoahHeap::is_in_active_generation(oop obj) const {
   // No flickering!
   assert(gen == active_generation(), "Race?");
 
-  switch (_affiliations[index]) {
+  switch (region_affiliation(index)) {
   case ShenandoahAffiliation::FREE:
     // Free regions are in old, young, and global collections
     return true;
@@ -382,25 +382,25 @@ inline bool ShenandoahHeap::is_in_active_generation(oop obj) const {
     // Old regions are in old and global collections, not in young collections
     return !gen->is_young();
   default:
-    assert(false, "Bad affiliation (%d) for region " SIZE_FORMAT, _affiliations[index], index);
+    assert(false, "Bad affiliation (%d) for region " SIZE_FORMAT, region_affiliation(index), index);
     return false;
   }
 }
 
 inline bool ShenandoahHeap::is_in_young(const void* p) const {
-  return is_in_reserved(p) && (_affiliations[heap_region_index_containing(p)] == ShenandoahAffiliation::YOUNG_GENERATION);
+  return is_in_reserved(p) && (region_affiliation(heap_region_index_containing(p)) == ShenandoahAffiliation::YOUNG_GENERATION);
 }
 
 inline bool ShenandoahHeap::is_in_old(const void* p) const {
-  return is_in_reserved(p) && (_affiliations[heap_region_index_containing(p)] == ShenandoahAffiliation::OLD_GENERATION);
+  return is_in_reserved(p) && (region_affiliation(heap_region_index_containing(p)) == ShenandoahAffiliation::OLD_GENERATION);
 }
 
 inline bool ShenandoahHeap::is_in_old_during_young_collection(oop obj) const {
   return active_generation()->is_young() && is_in_old(obj);
 }
 
-inline ShenandoahAffiliation ShenandoahHeap::region_affiliation(const ShenandoahHeapRegion *r) {
-  return (ShenandoahAffiliation) _affiliations[r->index()];
+inline ShenandoahAffiliation ShenandoahHeap::region_affiliation(const ShenandoahHeapRegion *r) const {
+  return region_affiliation(r->index());
 }
 
 inline void ShenandoahHeap::assert_lock_for_affiliation(ShenandoahAffiliation orig_affiliation,
@@ -419,7 +419,7 @@ inline void ShenandoahHeap::assert_lock_for_affiliation(ShenandoahAffiliation or
   //
   // Note: during full GC, all transitions between states are possible.  During Full GC, we should be in a safepoint.
 
-  if ((orig_affiliation == ShenandoahAffiliation::FREE) || (new_affiliation == ShenandoahAffiliation::FREE)) {
+  if (orig_affiliation == ShenandoahAffiliation::FREE) {
     shenandoah_assert_heaplocked_or_safepoint();
   }
 }
@@ -428,11 +428,11 @@ inline void ShenandoahHeap::set_affiliation(ShenandoahHeapRegion* r, ShenandoahA
 #ifdef ASSERT
   assert_lock_for_affiliation(region_affiliation(r), new_affiliation);
 #endif
-  _affiliations[r->index()] = (uint8_t) new_affiliation;
+  Atomic::store(_affiliations + r->index(), (uint8_t) new_affiliation);
 }
 
-inline ShenandoahAffiliation ShenandoahHeap::region_affiliation(size_t index) {
-  return (ShenandoahAffiliation) _affiliations[index];
+inline ShenandoahAffiliation ShenandoahHeap::region_affiliation(size_t index) const {
+  return (ShenandoahAffiliation) Atomic::load(_affiliations + index);
 }
 
 inline bool ShenandoahHeap::requires_marking(const void* entry) const {

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp

@@ -88,19 +88,20 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c
   if (ZapUnusedHeapArea && committed) {
     SpaceMangler::mangle_region(MemRegion(_bottom, _end));
   }
+  _recycling.unset();
 }
 
 void ShenandoahHeapRegion::report_illegal_transition(const char *method) {
   stringStream ss;
-  ss.print("Illegal region state transition from \"%s\", at %s\n  ", region_state_to_string(_state), method);
+  ss.print("Illegal region state transition from \"%s\", at %s\n  ", region_state_to_string(state()), method);
   print_on(&ss);
   fatal("%s", ss.freeze());
 }
 
 void ShenandoahHeapRegion::make_regular_allocation(ShenandoahAffiliation affiliation) {
   shenandoah_assert_heaplocked();
   reset_age();
-  switch (_state) {
+  switch (state()) {
     case _empty_uncommitted:
       do_commit();
     case _empty_committed:
@@ -120,7 +121,7 @@ void ShenandoahHeapRegion::make_regular_allocation(ShenandoahAffiliation affilia
 void ShenandoahHeapRegion::make_affiliated_maybe() {
   shenandoah_assert_heaplocked();
   assert(!ShenandoahHeap::heap()->mode()->is_generational(), "Only call if non-generational");
-  switch (_state) {
+  switch (state()) {
    case _empty_uncommitted:
    case _empty_committed:
    case _cset:
@@ -146,14 +147,15 @@ void ShenandoahHeapRegion::make_regular_bypass() {
           ShenandoahHeap::heap()->is_degenerated_gc_in_progress(),
           "Only for STW GC or when Universe is initializing (CDS)");
   reset_age();
-  switch (_state) {
+  auto cur_state = state();
+  switch (cur_state) {
     case _empty_uncommitted:
       do_commit();
     case _empty_committed:
     case _cset:
     case _humongous_start:
     case _humongous_cont:
-      if (_state == _humongous_start || _state == _humongous_cont) {
+      if (cur_state == _humongous_start || cur_state == _humongous_cont) {
         // CDS allocates chunks of the heap to fill with regular objects. The allocator
         // will dutifully track any waste in the unused portion of the last region. Once
         // CDS has finished initializing the objects, it will convert these regions to
@@ -177,7 +179,7 @@ void ShenandoahHeapRegion::make_regular_bypass() {
 void ShenandoahHeapRegion::make_humongous_start() {
   shenandoah_assert_heaplocked();
   reset_age();
-  switch (_state) {
+  switch (state()) {
     case _empty_uncommitted:
       do_commit();
     case _empty_committed:
@@ -194,7 +196,7 @@ void ShenandoahHeapRegion::make_humongous_start_bypass(ShenandoahAffiliation aff
   // Don't bother to account for affiliated regions during Full GC.  We recompute totals at end.
   set_affiliation(affiliation);
   reset_age();
-  switch (_state) {
+  switch (state()) {
     case _empty_committed:
     case _regular:
     case _humongous_start:
@@ -209,7 +211,7 @@ void ShenandoahHeapRegion::make_humongous_start_bypass(ShenandoahAffiliation aff
 void ShenandoahHeapRegion::make_humongous_cont() {
   shenandoah_assert_heaplocked();
   reset_age();
-  switch (_state) {
+  switch (state()) {
     case _empty_uncommitted:
       do_commit();
     case _empty_committed:
@@ -226,7 +228,7 @@ void ShenandoahHeapRegion::make_humongous_cont_bypass(ShenandoahAffiliation affi
   set_affiliation(affiliation);
   // Don't bother to account for affiliated regions during Full GC.  We recompute totals at end.
   reset_age();
-  switch (_state) {
+  switch (state()) {
     case _empty_committed:
     case _regular:
     case _humongous_start:
@@ -242,7 +244,7 @@ void ShenandoahHeapRegion::make_pinned() {
   shenandoah_assert_heaplocked();
   assert(pin_count() > 0, "Should have pins: " SIZE_FORMAT, pin_count());
 
-  switch (_state) {
+  switch (state()) {
     case _regular:
       set_state(_pinned);
     case _pinned_cset:
@@ -253,7 +255,7 @@ void ShenandoahHeapRegion::make_pinned() {
     case _pinned_humongous_start:
       return;
     case _cset:
-      _state = _pinned_cset;
+      set_state(_pinned_cset);
       return;
     default:
       report_illegal_transition("pinning");
@@ -264,7 +266,7 @@ void ShenandoahHeapRegion::make_unpinned() {
   shenandoah_assert_heaplocked();
   assert(pin_count() == 0, "Should not have pins: " SIZE_FORMAT, pin_count());
 
-  switch (_state) {
+  switch (state()) {
     case _pinned:
       assert(is_affiliated(), "Pinned region should be affiliated");
       set_state(_regular);
@@ -286,7 +288,7 @@ void ShenandoahHeapRegion::make_unpinned() {
 void ShenandoahHeapRegion::make_cset() {
   shenandoah_assert_heaplocked();
   // Leave age untouched.  We need to consult the age when we are deciding whether to promote evacuated objects.
-  switch (_state) {
+  switch (state()) {
     case _regular:
       set_state(_cset);
     case _cset:
@@ -299,7 +301,7 @@ void ShenandoahHeapRegion::make_cset() {
 void ShenandoahHeapRegion::make_trash() {
   shenandoah_assert_heaplocked();
   reset_age();
-  switch (_state) {
+  switch (state()) {
     case _humongous_start:
     case _humongous_cont:
     {
@@ -329,10 +331,9 @@ void ShenandoahHeapRegion::make_trash_immediate() {
 }
 
 void ShenandoahHeapRegion::make_empty() {
-  shenandoah_assert_heaplocked();
   reset_age();
   CENSUS_NOISE(clear_youth();)
-  switch (_state) {
+  switch (state()) {
     case _trash:
       set_state(_empty_committed);
       _empty_time = os::elapsedTime();
@@ -344,7 +345,7 @@ void ShenandoahHeapRegion::make_empty() {
 
 void ShenandoahHeapRegion::make_uncommitted() {
   shenandoah_assert_heaplocked();
-  switch (_state) {
+  switch (state()) {
     case _empty_committed:
       do_uncommit();
       set_state(_empty_uncommitted);
@@ -358,7 +359,7 @@ void ShenandoahHeapRegion::make_committed_bypass() {
   shenandoah_assert_heaplocked();
   assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC");
 
-  switch (_state) {
+  switch (state()) {
     case _empty_uncommitted:
       do_commit();
       set_state(_empty_committed);
@@ -399,7 +400,7 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const {
   st->print("|");
   st->print(SIZE_FORMAT_W(5), this->_index);
 
-  switch (_state) {
+  switch (state()) {
     case _empty_uncommitted:
       st->print("|EU ");
       break;
@@ -569,27 +570,62 @@ ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const {
   return r;
 }
 
-void ShenandoahHeapRegion::recycle() {
-  shenandoah_assert_heaplocked();
-  ShenandoahHeap* heap = ShenandoahHeap::heap();
-  ShenandoahGeneration* generation = heap->generation_for(affiliation());
 
-  heap->decrease_used(generation, used());
-  generation->decrement_affiliated_region_count();
+void ShenandoahHeapRegion::recycle_internal() {
+  assert(_recycling.is_set() && is_trash(), "Wrong state");
+  ShenandoahHeap* heap = ShenandoahHeap::heap();
 
   set_top(bottom());
   clear_live_data();
   reset_alloc_metadata();
-
   heap->marking_context()->reset_top_at_mark_start(this);
-
   set_update_watermark(bottom());
+  if (ZapUnusedHeapArea) {
+    SpaceMangler::mangle_region(MemRegion(bottom(), end()));
+  }
 
   make_empty();
-
   set_affiliation(FREE);
-  if (ZapUnusedHeapArea) {
-    SpaceMangler::mangle_region(MemRegion(bottom(), end()));
+}
+
+void ShenandoahHeapRegion::try_recycle_under_lock() {
+  shenandoah_assert_heaplocked();
+  if (is_trash() && _recycling.try_set()) {
+    if (is_trash()) {
+      ShenandoahHeap* heap = ShenandoahHeap::heap();
+      ShenandoahGeneration* generation = heap->generation_for(affiliation());
+
+      heap->decrease_used(generation, used());
+      generation->decrement_affiliated_region_count();
+
+      recycle_internal();
+    }
+    _recycling.unset();
+  } else {
+    // Ensure recycling is unset before returning to mutator to continue memory allocation.
+    while (_recycling.is_set()) {
+      if (os::is_MP()) {
+        SpinPause();
+      } else {
+        os::naked_yield();
+      }
+    }
+  }
+}
+
+void ShenandoahHeapRegion::try_recycle() {
+  shenandoah_assert_not_heaplocked();
+  if (is_trash() && _recycling.try_set()) {
+    // Double check region state after win the race to set recycling flag
+    if (is_trash()) {
+      ShenandoahHeap* heap = ShenandoahHeap::heap();
+      ShenandoahGeneration* generation = heap->generation_for(affiliation());
+      heap->decrease_used(generation, used());
+      generation->decrement_affiliated_region_count_without_lock();
+
+      recycle_internal();
+    }
+    _recycling.unset();
   }
 }
 
@@ -795,11 +831,11 @@ void ShenandoahHeapRegion::set_state(RegionState to) {
     evt.set_index((unsigned) index());
     evt.set_start((uintptr_t)bottom());
     evt.set_used(used());
-    evt.set_from(_state);
+    evt.set_from(state());
     evt.set_to(to);
     evt.commit();
   }
-  _state = to;
+  Atomic::store(&_state, to);
 }
 
 void ShenandoahHeapRegion::record_pin() {

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -165,6 +165,7 @@ class ShenandoahHeapRegion {
   }
 
   void report_illegal_transition(const char* method);
+  void recycle_internal();
 
 public:
   static int region_states_num() {
@@ -188,33 +189,35 @@ class ShenandoahHeapRegion {
   void make_uncommitted();
   void make_committed_bypass();
 
-  // Individual states:
-  bool is_empty_uncommitted()      const { return _state == _empty_uncommitted; }
-  bool is_empty_committed()        const { return _state == _empty_committed; }
-  bool is_regular()                const { return _state == _regular; }
-  bool is_humongous_continuation() const { return _state == _humongous_cont; }
-
-  // Participation in logical groups:
-  bool is_empty()                  const { return is_empty_committed() || is_empty_uncommitted(); }
-  bool is_active()                 const { return !is_empty() && !is_trash(); }
-  bool is_trash()                  const { return _state == _trash; }
-  bool is_humongous_start()        const { return _state == _humongous_start || _state == _pinned_humongous_start; }
-  bool is_humongous()              const { return is_humongous_start() || is_humongous_continuation(); }
+  // Primitive state predicates
+  bool is_empty_uncommitted()      const { return state() == _empty_uncommitted; }
+  bool is_empty_committed()        const { return state() == _empty_committed; }
+  bool is_regular()                const { return state() == _regular; }
+  bool is_humongous_continuation() const { return state() == _humongous_cont; }
+  bool is_regular_pinned()         const { return state() == _pinned; }
+  bool is_trash()                  const { return state() == _trash; }
+
+  // Derived state predicates (boolean combinations of individual states)
+  bool static is_empty_state(RegionState state) { return state == _empty_committed || state == _empty_uncommitted; }
+  bool static is_humongous_start_state(RegionState state) { return state == _humongous_start || state == _pinned_humongous_start; }
+  bool is_empty()                  const { return is_empty_state(this->state()); }
+  bool is_active()                 const { auto cur_state = state(); return !is_empty_state(cur_state) && cur_state != _trash; }
+  bool is_humongous_start()        const { return is_humongous_start_state(state()); }
+  bool is_humongous()              const { auto cur_state = state(); return is_humongous_start_state(cur_state) || cur_state == _humongous_cont; }
   bool is_committed()              const { return !is_empty_uncommitted(); }
-  bool is_cset()                   const { return _state == _cset   || _state == _pinned_cset; }
-  bool is_pinned()                 const { return _state == _pinned || _state == _pinned_cset || _state == _pinned_humongous_start; }
-  bool is_regular_pinned()         const { return _state == _pinned; }
+  bool is_cset()                   const { auto cur_state = state(); return cur_state == _cset || cur_state == _pinned_cset; }
+  bool is_pinned()                 const { auto cur_state = state(); return cur_state == _pinned || cur_state == _pinned_cset || cur_state == _pinned_humongous_start; }
 
   inline bool is_young() const;
   inline bool is_old() const;
   inline bool is_affiliated() const;
 
   // Macro-properties:
-  bool is_alloc_allowed()          const { return is_empty() || is_regular() || _state == _pinned; }
-  bool is_stw_move_allowed()       const { return is_regular() || _state == _cset || (ShenandoahHumongousMoves && _state == _humongous_start); }
+  bool is_alloc_allowed()          const { auto cur_state = state(); return is_empty_state(cur_state) || cur_state == _regular || cur_state == _pinned; }
+  bool is_stw_move_allowed()       const { auto cur_state = state(); return cur_state == _regular || cur_state == _cset || (ShenandoahHumongousMoves && cur_state == _humongous_start); }
 
-  RegionState state()              const { return _state; }
-  int  state_ordinal()             const { return region_state_to_ordinal(_state); }
+  RegionState state()              const { return Atomic::load(&_state); }
+  int  state_ordinal()             const { return region_state_to_ordinal(state()); }
 
   void record_pin();
   void record_unpin();
@@ -243,7 +246,7 @@ class ShenandoahHeapRegion {
   HeapWord* _top_before_promoted;
 
   // Seldom updated fields
-  RegionState _state;
+  volatile RegionState _state;
   HeapWord* _coalesce_and_fill_boundary; // for old regions not selected as collection set candidates.
 
   // Frequently updated fields
@@ -261,6 +264,8 @@ class ShenandoahHeapRegion {
   uint _age;
   CENSUS_NOISE(uint _youth;)   // tracks epochs of retrograde ageing (rejuvenation)
 
+  ShenandoahSharedFlag _recycling; // Used to indicate that the region is being recycled; see try_recycle*().
+
 public:
   ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed);
 
@@ -376,7 +381,9 @@ class ShenandoahHeapRegion {
 
   void print_on(outputStream* st) const;
 
-  void recycle();
+  void try_recycle_under_lock();
+
+  void try_recycle();
 
   inline void begin_preemptible_coalesce_and_fill() {
     _coalesce_and_fill_boundary = _bottom;

src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp

@@ -74,3 +74,7 @@ uint ShenandoahWorkerPolicy::calc_workers_for_final_update_ref() {
 uint ShenandoahWorkerPolicy::calc_workers_for_conc_reset() {
   return ConcGCThreads;
 }
+
+uint ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup() {
+  return ConcGCThreads;
+}

src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp

@@ -64,6 +64,9 @@ class ShenandoahWorkerPolicy : AllStatic {
 
   // Calculate workers for concurrent reset
   static uint calc_workers_for_conc_reset();
+
+  // Calculate workers for concurrent cleanup
+  static uint calc_workers_for_conc_cleanup();
 };
 
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHWORKERPOLICY_HPP

src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp

@@ -39,7 +39,7 @@
   volatile_nonstatic_field(ShenandoahGeneration, _used,            size_t)                            \
   static_field(ShenandoahHeapRegion, RegionSizeBytes,              size_t)                            \
   static_field(ShenandoahHeapRegion, RegionSizeBytesShift,         size_t)                            \
-  nonstatic_field(ShenandoahHeapRegion, _state,                    ShenandoahHeapRegion::RegionState) \
+  volatile_nonstatic_field(ShenandoahHeapRegion, _state,           ShenandoahHeapRegion::RegionState) \
   nonstatic_field(ShenandoahHeapRegion, _index,                    size_t const)                      \
   nonstatic_field(ShenandoahHeapRegion, _bottom,                   HeapWord* const)                   \
   nonstatic_field(ShenandoahHeapRegion, _top,                      HeapWord*)                         \