8348594: Shenandoah: Do not penalize for degeneration when not the fault of triggering heuristic

src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp

@@ -240,6 +240,10 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() {
   log_debug(gc)("should_start_gc? available: %zu, soft_max_capacity: %zu"
                 ", allocated: %zu", available, capacity, allocated);
 
+  if (_start_gc_is_pending) {
+    return true;
+  }
+
   // Track allocation rate even if we decide to start a cycle for other reasons.
   double rate = _allocation_rate.sample(allocated);
   _last_trigger = OTHER;
@@ -249,6 +253,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() {
     log_trigger("Free (%zu%s) is below minimum threshold (%zu%s)",
                  byte_size_in_proper_unit(available), proper_unit_for_byte_size(available),
                  byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold));
+    accept_trigger_with_type(OTHER);
     return true;
   }
 
@@ -261,6 +266,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() {
                    _gc_times_learned + 1, max_learn,
                    byte_size_in_proper_unit(available), proper_unit_for_byte_size(available),
                    byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold));
+      accept_trigger_with_type(OTHER);
       return true;
     }
   }
@@ -292,7 +298,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() {
                        byte_size_in_proper_unit(spike_headroom),      proper_unit_for_byte_size(spike_headroom),
                        byte_size_in_proper_unit(penalties),           proper_unit_for_byte_size(penalties),
                        byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom));
-    _last_trigger = RATE;
+    accept_trigger_with_type(RATE);
     return true;
   }
 
@@ -303,11 +309,16 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() {
                  byte_size_in_proper_unit(rate), proper_unit_for_byte_size(rate),
                  byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom),
                  _spike_threshold_sd);
-    _last_trigger = SPIKE;
+    accept_trigger_with_type(SPIKE);
     return true;
   }
 
-  return ShenandoahHeuristics::should_start_gc();
+  if (ShenandoahHeuristics::should_start_gc()) {
+    _start_gc_is_pending = true;
+    return true;
+  } else {
+    return false;
+  }
 }
 
 void ShenandoahAdaptiveHeuristics::adjust_last_trigger_parameters(double amount) {

src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp

@@ -145,6 +145,11 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics {
   // this threshold, or we might consider this when dynamically resizing generations
   // in the generational case. Controlled by global flag ShenandoahMinFreeThreshold.
   size_t min_free_threshold();
+
+  inline void accept_trigger_with_type(Trigger trigger_type) {
+    _last_trigger = trigger_type;
+    ShenandoahHeuristics::accept_trigger();
+  }
 };
 
 #endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHADAPTIVEHEURISTICS_HPP

src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp

@@ -52,6 +52,7 @@ void ShenandoahAggressiveHeuristics::choose_collection_set_from_regiondata(Shena
 
 bool ShenandoahAggressiveHeuristics::should_start_gc() {
   log_trigger("Start next cycle immediately");
+  accept_trigger();
   return true;
 }
 

src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp

@@ -61,6 +61,7 @@ bool ShenandoahCompactHeuristics::should_start_gc() {
     log_trigger("Free (%zu%s) is below minimum threshold (%zu%s)",
                 byte_size_in_proper_unit(available),     proper_unit_for_byte_size(available),
                 byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold));
+    accept_trigger();
     return true;
   }
 
@@ -69,6 +70,7 @@ bool ShenandoahCompactHeuristics::should_start_gc() {
     log_trigger("Allocated since last cycle (%zu%s) is larger than allocation threshold (%zu%s)",
                 byte_size_in_proper_unit(bytes_allocated),           proper_unit_for_byte_size(bytes_allocated),
                 byte_size_in_proper_unit(threshold_bytes_allocated), proper_unit_for_byte_size(threshold_bytes_allocated));
+    accept_trigger();
     return true;
   }
 

src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp

@@ -46,6 +46,9 @@ int ShenandoahHeuristics::compare_by_garbage(RegionData a, RegionData b) {
 }
 
 ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahSpaceInfo* space_info) :
+  _start_gc_is_pending(false),
+  _declined_trigger_count(0),
+  _most_recent_declined_trigger_count(0),
   _space_info(space_info),
   _region_data(nullptr),
   _guaranteed_gc_interval(0),
@@ -184,10 +187,14 @@ void ShenandoahHeuristics::record_cycle_end() {
 }
 
 bool ShenandoahHeuristics::should_start_gc() {
+  if (_start_gc_is_pending) {
+    return true;
+  }
   // Perform GC to cleanup metaspace
   if (has_metaspace_oom()) {
     // Some of vmTestbase/metaspace tests depend on following line to count GC cycles
     log_trigger("%s", GCCause::to_string(GCCause::_metadata_GC_threshold));
+    accept_trigger();
     return true;
   }
 
@@ -196,10 +203,11 @@ bool ShenandoahHeuristics::should_start_gc() {
     if (last_time_ms > _guaranteed_gc_interval) {
       log_trigger("Time since last GC (%.0f ms) is larger than guaranteed interval (%zu ms)",
                    last_time_ms, _guaranteed_gc_interval);
+      accept_trigger();
       return true;
     }
   }
-
+  decline_trigger();
   return false;
 }
 
@@ -211,6 +219,12 @@ void ShenandoahHeuristics::adjust_penalty(intx step) {
   assert(0 <= _gc_time_penalties && _gc_time_penalties <= 100,
          "In range before adjustment: %zd", _gc_time_penalties);
 
+  if ((_most_recent_declined_trigger_count <= Penalty_Free_Declinations) && (step > 0)) {
+    // Don't penalize if heuristics are not responsible for a negative outcome.  Allow Penalty_Free_Declinations following
+    // previous GC for self calibration without penalty.
+    step = 0;
+  }
+
   intx new_val = _gc_time_penalties + step;
   if (new_val < 0) {
     new_val = 0;

src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp

@@ -69,6 +69,9 @@ class ShenandoahHeuristics : public CHeapObj<mtGC> {
   static const intx Degenerated_Penalty = 10; // how much to penalize average GC duration history on Degenerated GC
   static const intx Full_Penalty        = 20; // how much to penalize average GC duration history on Full GC
 
+  // How many times can I decline a trigger opportunity without being penalized for excessive idle span before trigger?
+  static const size_t Penalty_Free_Declinations = 16;
+
 #ifdef ASSERT
   enum UnionTag {
     is_uninitialized, is_garbage, is_live_data
@@ -78,6 +81,18 @@ class ShenandoahHeuristics : public CHeapObj<mtGC> {
 protected:
   static const uint Moving_Average_Samples = 10; // Number of samples to store in moving averages
 
+  bool _start_gc_is_pending;              // True denotes that GC has been triggered, so no need to trigger again.
+  size_t _declined_trigger_count;         // This counts how many times since previous GC finished that this
+                                          //  heuristic has answered false to should_start_gc().
+  size_t _most_recent_declined_trigger_count;
+                                       ;  // This represents the value of _declined_trigger_count as captured at the
+                                          //  moment the most recent GC effort was triggered.  In case the most recent
+                                          //  concurrent GC effort degenerates, the value of this variable allows us to
+                                          //  differentiate between degeneration because heuristic was overly optimistic
+                                          //  in delaying the trigger vs. degeneration for other reasons (such as the
+                                          //  most recent GC triggered "immediately" after previous GC finished, but the
+                                          //  free headroom has already been depleted).
+
   class RegionData {
     private:
     ShenandoahHeapRegion* _region;
@@ -167,6 +182,16 @@ class ShenandoahHeuristics : public CHeapObj<mtGC> {
 
   void adjust_penalty(intx step);
 
+  inline void accept_trigger() {
+    _most_recent_declined_trigger_count = _declined_trigger_count;
+    _declined_trigger_count = 0;
+    _start_gc_is_pending = true;
+  }
+
+  inline void decline_trigger() {
+    _declined_trigger_count++;
+  }
+
 public:
   ShenandoahHeuristics(ShenandoahSpaceInfo* space_info);
   virtual ~ShenandoahHeuristics();
@@ -185,6 +210,10 @@ class ShenandoahHeuristics : public CHeapObj<mtGC> {
 
   virtual bool should_start_gc();
 
+  inline void cancel_trigger_request() {
+    _start_gc_is_pending = false;
+  }
+
   virtual bool should_degenerate_cycle();
 
   virtual void record_success_concurrent();

src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp

@@ -36,6 +36,7 @@ ShenandoahPassiveHeuristics::ShenandoahPassiveHeuristics(ShenandoahSpaceInfo* sp
 
 bool ShenandoahPassiveHeuristics::should_start_gc() {
   // Never do concurrent GCs.
+  decline_trigger();
   return false;
 }
 

src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp

@@ -55,6 +55,7 @@ bool ShenandoahStaticHeuristics::should_start_gc() {
     log_trigger("Free (%zu%s) is below minimum threshold (%zu%s)",
                  byte_size_in_proper_unit(available),           proper_unit_for_byte_size(available),
                  byte_size_in_proper_unit(threshold_available), proper_unit_for_byte_size(threshold_available));
+    accept_trigger();
     return true;
   }
   return ShenandoahHeuristics::should_start_gc();

src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp

@@ -119,6 +119,8 @@ bool ShenandoahYoungHeuristics::should_start_gc() {
     if (old_generation->is_preparing_for_mark() || old_generation->is_concurrent_mark_in_progress()) {
       size_t old_time_elapsed = size_t(old_heuristics->elapsed_cycle_time() * 1000);
       if (old_time_elapsed < ShenandoahMinimumOldTimeMs) {
+        // Do not decline_trigger() when waiting for minimum quantum of Old-gen marking.  It is not at our discretion
+        // to trigger at this time.
         return false;
       }
     }
@@ -140,6 +142,7 @@ bool ShenandoahYoungHeuristics::should_start_gc() {
     // Detect unsigned arithmetic underflow
     assert(promo_potential < heap->capacity(), "Sanity");
     log_trigger("Expedite promotion of " PROPERFMT, PROPERFMTARGS(promo_potential));
+    accept_trigger();
     return true;
   }
 
@@ -150,9 +153,11 @@ bool ShenandoahYoungHeuristics::should_start_gc() {
     // candidates, but has not completed. There is no point in trying to start the young cycle before the old
     // cycle completes.
     log_trigger("Expedite mixed evacuation of %zu regions", mixed_candidates);
+    accept_trigger();
     return true;
   }
 
+  // Don't decline_trigger() here  That was done in ShenandoahAdaptiveHeuristics::should_start_gc()
   return false;
 }
 

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -947,11 +947,25 @@ class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
 void ShenandoahConcurrentGC::op_weak_roots() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
-  // Concurrent weak root processing
-  ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
-  ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
-  ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
-  heap->workers()->run_task(&task);
+  {
+    // Concurrent weak root processing
+    ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
+    ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
+    ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
+    heap->workers()->run_task(&task);
+  }
+
+  {
+    // It is possible for mutators executing the load reference barrier to have
+    // loaded an oop through a weak handle that has since been nulled out by
+    // weak root processing. Handshaking here forces them to complete the
+    // barrier before the GC cycle continues and does something that would
+    // change the evaluation of the barrier (for example, resetting the TAMS
+    // on trashed regions could make an oop appear to be marked _after_ the
+    // region has been recycled).
+    ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
+    heap->rendezvous_threads("Shenandoah Concurrent Weak Roots");
+  }
 }
 
 void ShenandoahConcurrentGC::op_class_unloading() {

src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp

@@ -131,6 +131,8 @@ void ShenandoahControlThread::run_service() {
       // GC is starting, bump the internal ID
       update_gc_id();
 
+      heuristics->cancel_trigger_request();
+
       heap->reset_bytes_allocated_since_gc_start();
 
       MetaspaceCombinedStats meta_sizes = MetaspaceUtils::get_combined_statistics();

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp

@@ -437,6 +437,7 @@ class ShenandoahFreeSet : public CHeapObj<mtGC> {
 
   inline size_t capacity()  const { return _partitions.capacity_of(ShenandoahFreeSetPartitionId::Mutator); }
   inline size_t used()      const { return _partitions.used_by(ShenandoahFreeSetPartitionId::Mutator);     }
+
   inline size_t available() const {
     assert(used() <= capacity(), "must use less than capacity");
     return capacity() - used();

src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp

@@ -65,21 +65,25 @@ void ShenandoahRegulatorThread::regulate_young_and_old_cycles() {
         if (request_concurrent_gc(GLOBAL)) {
           // Some of vmTestbase/metaspace tests depend on following line to count GC cycles
           _global_heuristics->log_trigger("%s", GCCause::to_string(GCCause::_metadata_GC_threshold));
+          _global_heuristics->cancel_trigger_request();
         }
       } else {
         if (_young_heuristics->should_start_gc()) {
           // Give the old generation a chance to run. The old generation cycle
           // begins with a 'bootstrap' cycle that will also collect young.
           if (start_old_cycle()) {
             log_debug(gc)("Heuristics request for old collection accepted");
+            _young_heuristics->cancel_trigger_request();
           } else if (request_concurrent_gc(YOUNG)) {
             log_debug(gc)("Heuristics request for young collection accepted");
+            _young_heuristics->cancel_trigger_request();
           }
         }
       }
     } else if (mode == ShenandoahGenerationalControlThread::servicing_old) {
       if (start_young_cycle()) {
         log_debug(gc)("Heuristics request to interrupt old for young collection accepted");
+        _young_heuristics->cancel_trigger_request();
       }
     }
 
@@ -93,8 +97,10 @@ void ShenandoahRegulatorThread::regulate_young_and_global_cycles() {
     if (_control_thread->gc_mode() == ShenandoahGenerationalControlThread::none) {
       if (start_global_cycle()) {
         log_debug(gc)("Heuristics request for global collection accepted.");
+        _global_heuristics->cancel_trigger_request();
       } else if (start_young_cycle()) {
         log_debug(gc)("Heuristics request for young collection accepted.");
+        _young_heuristics->cancel_trigger_request();
       }
     }