8312116 GenShen: make instantaneous allocation rate triggers more timely

src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp

@@ -30,7 +30,9 @@
 #include "memory/allocation.hpp"
 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
 #include "gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp"
+#include "gc/shenandoah/shenandoahFreeSet.hpp"
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
+#include "gc/shenandoah/shenandoahRegulatorThread.hpp"
 #include "gc/shenandoah/shenandoahSharedVariables.hpp"
 #include "utilities/numberSeq.hpp"
 
@@ -43,6 +45,12 @@ class ShenandoahAllocationRate : public CHeapObj<mtGC> {
 
   double upper_bound(double sds) const;
   bool is_spiking(double rate, double threshold) const;
+  double interval() const {
+    return _interval_sec;
+  }
+  double last_sample_time() const {
+    return _last_sample_time;
+  }
  private:
 
   double instantaneous_rate(double time, size_t allocated) const;
@@ -71,17 +79,34 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics {
 
   virtual ~ShenandoahAdaptiveHeuristics();
 
+  virtual void initialize();
+
   virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset,
                                                      RegionData* data, size_t size,
                                                      size_t actual_free);
 
+  virtual void adjust_penalty(intx step);
+
+  // In case we rebuild free set during idle span, such as when we finish OLD GC marking and add immediate garbage to
+  // free set, invoke this to recalibrate the triggering heuristic.
+  void resume_idle_span(size_t mutator_available);
+
+  void start_evac_span(size_t mutator_available);
+
+  // How much memory is available for mutator allocations?
+  //  (as calculated by mutator free at last rebuild minus mutator allocations since last rebuild)
+  inline size_t allocatable() const {
+    size_t total_allocations = _freeset->get_mutator_allocations();
+    return (total_allocations > _allocation_cliff)? 0: _allocation_cliff - total_allocations;
+  }
+
   void record_cycle_start();
+  void record_degenerated_cycle_start(bool out_of_cycle);
   void record_success_concurrent(bool abbreviated);
   void record_success_degenerated();
   void record_success_full();
 
   virtual bool should_start_gc();
-
   virtual const char* name()     { return "Adaptive"; }
   virtual bool is_diagnostic()   { return false; }
   virtual bool is_experimental() { return false; }
@@ -99,6 +124,9 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics {
   const static double LOWEST_EXPECTED_AVAILABLE_AT_END;
   const static double HIGHEST_EXPECTED_AVAILABLE_AT_END;
 
+  const static size_t GC_TIME_SAMPLE_SIZE;
+  const static size_t HISTORICAL_PERIOD_SAMPLE_SIZE;
+
   friend class ShenandoahAllocationRate;
 
   // Used to record the last trigger that signaled to start a GC.
@@ -113,6 +141,11 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics {
   void adjust_margin_of_error(double amount);
   void adjust_spike_threshold(double amount);
 
+  void add_rate_to_acceleration_history(double timestamp, double rate);
+  size_t accelerated_consumption(double& acceleration, double& current_rate, double predicted_cycle_time) const;
+
+  void start_idle_span();
+
 protected:
   ShenandoahAllocationRate _allocation_rate;
 
@@ -128,7 +161,7 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics {
   // rate exceeds this threshold, a GC cycle is started. As this value
   // decreases the sensitivity to allocation spikes increases. In other
   // words, lowering the spike threshold will tend to increase the number
-  // of concurrent GCs.
+  // of concurrent GCs because more scenarios will be seen as spiking.
   double _spike_threshold_sd;
 
   // Remember which trigger is responsible for the last GC cycle. When the
@@ -142,6 +175,47 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics {
   // source of feedback to adjust trigger parameters.
   TruncatedSeq _available;
 
+  ShenandoahFreeSet* _freeset;
+  ShenandoahRegulatorThread* _regulator_thread;
+
+  size_t _previous_total_allocations;
+  double _previous_allocation_timestamp;
+  size_t _total_allocations_at_start_of_idle;
+  size_t _allocation_cliff;
+
+  // Keep track of GC_TIME_SAMPLE_SIZE most recent concurrent GC cycle times
+  uint _gc_time_first_sample_index;
+  uint _gc_time_num_samples;
+  double* const _gc_time_timestamps;
+  double* const _gc_time_samples;
+  double* const _gc_time_xy;    // timestamp * sample
+  double* const _gc_time_xx;    // timestamp squared
+  double _gc_time_sum_of_timestamps;
+  double _gc_time_sum_of_samples;
+  double _gc_time_sum_of_xy;
+  double _gc_time_sum_of_xx;
+
+  double _gc_time_m;            // slope
+  double _gc_time_b;            // y-intercept
+  double _gc_time_sd;           // sd on deviance from prediction
+
+  void add_gc_time(double timestamp_at_start, double duration);
+  void add_degenerated_gc_time(double timestamp_at_start, double duration);
+  double predict_gc_time(double timestamp_at_start);
+
+  // Keep track of SPIKE_ACCELERATION_SAMPLE_SIZE most recent spike allocation rate measurements. Note that it is
+  // typical to experience a small spike following end of GC cycle, as mutator threads refresh their TLABs.  But
+  // there is generally an abundance of memory at this time as well, so this will not generally trigger GC.
+  uint _spike_acceleration_first_sample_index;
+  uint _spike_acceleration_num_samples;
+  double* const _spike_acceleration_rate_samples;
+  double* const _spike_acceleration_rate_timestamps;
+
+  size_t _most_recent_headroom_at_start_of_idle;
+
+  double _acceleration_goodness_ratio;
+  size_t _consecutive_goodness;
+
   size_t min_free_threshold();
 };
 

src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp

@@ -47,7 +47,8 @@ ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahSpaceInfo* space_info) :
   _space_info(space_info),
   _region_data(nullptr),
   _guaranteed_gc_interval(0),
-  _cycle_start(os::elapsedTime()),
+  _precursor_cycle_start(os::elapsedTime()),
+  _cycle_start(_precursor_cycle_start),
   _last_cycle_end(0),
   _gc_times_learned(0),
   _gc_time_penalties(0),
@@ -180,6 +181,15 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec
                      collection_set->count());
 }
 
+void ShenandoahHeuristics::record_degenerated_cycle_start(bool out_of_cycle) {
+  if (out_of_cycle) {
+    _precursor_cycle_start = _cycle_start = os::elapsedTime();
+  } else {
+    _precursor_cycle_start = _cycle_start;
+    _cycle_start = os::elapsedTime();
+  }
+}
+
 void ShenandoahHeuristics::record_cycle_start() {
   _cycle_start = os::elapsedTime();
 }
@@ -288,3 +298,8 @@ void ShenandoahHeuristics::initialize() {
 double ShenandoahHeuristics::elapsed_cycle_time() const {
   return os::elapsedTime() - _cycle_start;
 }
+
+// Includes the time spent in abandoned concurrent GC cycle that pr
+double ShenandoahHeuristics::elapsed_degenerated_cycle_time() const {
+  return os::elapsedTime() - _precursor_cycle_start;
+}

src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp

@@ -99,6 +99,7 @@ class ShenandoahHeuristics : public CHeapObj<mtGC> {
 
   size_t _guaranteed_gc_interval;
 
+  double _precursor_cycle_start;
   double _cycle_start;
   double _last_cycle_end;
 
@@ -119,7 +120,7 @@ class ShenandoahHeuristics : public CHeapObj<mtGC> {
                                                      RegionData* data, size_t data_size,
                                                      size_t free) = 0;
 
-  void adjust_penalty(intx step);
+  virtual void adjust_penalty(intx step);
 
 public:
   ShenandoahHeuristics(ShenandoahSpaceInfo* space_info);
@@ -135,6 +136,8 @@ class ShenandoahHeuristics : public CHeapObj<mtGC> {
 
   virtual void record_cycle_start();
 
+  virtual void record_degenerated_cycle_start(bool out_of_cycle);
+
   virtual void record_cycle_end();
 
   virtual bool should_start_gc();
@@ -163,6 +166,7 @@ class ShenandoahHeuristics : public CHeapObj<mtGC> {
   virtual void initialize();
 
   double elapsed_cycle_time() const;
+  double elapsed_degenerated_cycle_time() const;
 };
 
 #endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHHEURISTICS_HPP

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -201,6 +201,7 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
     }
   }
 
+  size_t muator_free;
   if (heap->has_forwarded_objects()) {
     // Perform update-refs phase.
     vmop_entry_init_updaterefs();

src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp

@@ -110,7 +110,7 @@ void ShenandoahControlThread::run_service() {
   ShenandoahCollectorPolicy* policy = heap->shenandoah_policy();
 
   // Heuristics are notified of allocation failures here and other outcomes
-  // of the cycle. They're also used here to control whether the Nth consecutive
+  // of the cycle. They are also used here to control whether the Nth consecutive
   // degenerated cycle should be 'promoted' to a full cycle. The decision to
   // trigger a cycle or not is evaluated on the regulator thread.
   ShenandoahHeuristics* global_heuristics = heap->global_generation()->heuristics();
@@ -163,12 +163,11 @@ void ShenandoahControlThread::run_service() {
         set_gc_mode(stw_degenerated);
       } else {
         // TODO: if humongous_alloc_failure_pending, there might be value in trying a "compacting" degen before
-        // going all the way to full.  But it's a lot of work to implement this, and it may not provide value.
+        // going all the way to full.  But it is a lot of work to implement this, and it may not provide value.
         // A compacting degen can move young regions around without doing full old-gen mark (relying upon the
         // remembered set scan), so it might be faster than a full gc.
         //
         // Longer term, think about how to defragment humongous memory concurrently.
-
         heuristics->record_allocation_failure_gc();
         policy->record_alloc_failure_to_full();
         generation = select_global_generation();
@@ -213,7 +212,7 @@ void ShenandoahControlThread::run_service() {
       if (_requested_gc_cause == GCCause::_shenandoah_concurrent_gc) {
         if (_requested_generation == OLD && heap->doing_mixed_evacuations()) {
           // If a request to start an old cycle arrived while an old cycle was running, but _before_
-          // it chose any regions for evacuation we don't want to start a new old cycle. Rather, we want
+          // it chose any regions for evacuation we do not want to start a new old cycle. Rather, we want
           // the heuristic to run a young collection so that we can evacuate some old regions.
           assert(!heap->is_concurrent_old_mark_in_progress(), "Should not be running mixed collections and concurrent marking");
           generation = YOUNG;
@@ -224,7 +223,7 @@ void ShenandoahControlThread::run_service() {
         cause = GCCause::_shenandoah_concurrent_gc;
         set_gc_mode(default_mode);
 
-        // Don't start a new old marking if there is one already in progress
+        // Do not start a new old marking if there is one already in progress
         if (generation == OLD && heap->is_concurrent_old_mark_in_progress()) {
           set_gc_mode(servicing_old);
         }
@@ -235,7 +234,7 @@ void ShenandoahControlThread::run_service() {
           heap->set_unload_classes(false);
         }
 
-        // Don't want to spin in this loop and start a cycle every time, so
+        // Do not want to spin in this loop and start a cycle every time, so
         // clear requested gc cause. This creates a race with callers of the
         // blocking 'request_gc' method, but there it loops and resets the
         // '_requested_gc_cause' until a full cycle is completed.
@@ -399,7 +398,7 @@ void ShenandoahControlThread::run_service() {
       last_shrink_time = current;
     }
 
-    // Don't wait around if there was an allocation failure - start the next cycle immediately.
+    // Do not wait around if there was an allocation failure - start the next cycle immediately.
     if (!is_alloc_failure_gc()) {
       // The timed wait is necessary because this thread has a responsibility to send
       // 'alloc_words' to the pacer when it does not perform a GC.
@@ -408,7 +407,7 @@ void ShenandoahControlThread::run_service() {
     }
   }
 
-  // Wait for the actual stop(), can't leave run_service() earlier.
+  // Wait for the actual stop(), cannot leave run_service() earlier.
   while (!should_terminate()) {
     os::naked_short_sleep(ShenandoahControlIntervalMin);
   }
@@ -799,7 +798,7 @@ bool ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 
   GCIdMark gc_id_mark;
-  ShenandoahGCSession session(cause, _degen_generation);
+  ShenandoahDegeneratedGCSession session(cause, _degen_generation, point == ShenandoahGC::_degenerated_outside_cycle);
 
   ShenandoahDegenGC gc(point, _degen_generation);
   gc.collect(cause);
@@ -900,8 +899,6 @@ bool ShenandoahControlThread::request_concurrent_gc(ShenandoahGenerationType gen
   }
 
   if (preempt_old_marking(generation)) {
-    log_info(gc)("Preempting old generation mark to allow %s GC", shenandoah_generation_name(generation));
-    assert(gc_mode() == servicing_old, "Expected to be servicing old, but was: %s.", gc_mode_name(gc_mode()));
     _requested_gc_cause = GCCause::_shenandoah_concurrent_gc;
     _requested_generation = generation;
     _preemption_requested.set();