8327388: GenShen: census during marking is partial

src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp

@@ -125,9 +125,7 @@ void ShenandoahAgeCensus::prepare_for_census_update() {
 // Update the census data from appropriate sources,
 // and compute the new tenuring threshold.
 void ShenandoahAgeCensus::update_census(size_t age0_pop, AgeTable* pv1, AgeTable* pv2) {
-  // Check that we won't overwrite existing data: caller is
-  // responsible for explicitly clearing the slot via calling
-  // prepare_for_census_update().
+  prepare_for_census_update();
   assert(_global_age_table[_epoch]->is_clear(), "Dirty decks");
   CENSUS_NOISE(assert(_global_noise[_epoch].is_clear(), "Dirty decks");)
   if (ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) {
@@ -155,6 +153,10 @@ void ShenandoahAgeCensus::update_census(size_t age0_pop, AgeTable* pv1, AgeTable
   }
 
   update_tenuring_threshold();
+
+  // used for checking reasonableness of census coverage, non-product
+  // only.
+  NOT_PRODUCT(update_total();)
 }
 
 
@@ -213,6 +215,27 @@ bool ShenandoahAgeCensus::is_clear_local() {
   }
   return true;
 }
+
+size_t ShenandoahAgeCensus::get_all_ages(uint snap) {
+  assert(snap < MAX_SNAPSHOTS, "Out of bounds");
+  size_t pop = 0;
+  const AgeTable* pv = _global_age_table[snap];
+  for (uint i = 0; i < MAX_COHORTS; i++) {
+    pop += pv->sizes[i];
+  }
+  return pop;
+}
+
+size_t ShenandoahAgeCensus::get_skipped(uint snap) {
+  assert(snap < MAX_SNAPSHOTS, "Out of bounds");
+  return _global_noise[snap].skipped;
+}
+
+void ShenandoahAgeCensus::update_total() {
+  _counted = get_all_ages(_epoch);
+  _skipped = get_skipped(_epoch);
+  _total   = _counted + _skipped;
+}
 #endif // !PRODUCT
 
 void ShenandoahAgeCensus::update_tenuring_threshold() {

src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp

@@ -101,8 +101,16 @@ class ShenandoahAgeCensus: public CHeapObj<mtGC> {
 #ifdef SHENANDOAH_CENSUS_NOISE
   ShenandoahNoiseStats* _global_noise; // Noise stats, one per snapshot
   ShenandoahNoiseStats* _local_noise;  // Local scratch table for noise stats, one per worker
+
+  size_t _skipped;                   // net size of objects encountered, but skipped during census,
+                                     // because their age was indeterminate
 #endif // SHENANDOAH_CENSUS_NOISE
 
+#ifndef PRODUCT
+  size_t _counted;                   // net size of objects counted in census
+  size_t _total;                     // net size of objects encountered (counted or skipped) in census
+#endif
+
   uint _epoch;                       // Current epoch (modulo max age)
   uint *_tenuring_threshold;         // An array of the last N tenuring threshold values we
                                      // computed.
@@ -111,16 +119,45 @@ class ShenandoahAgeCensus: public CHeapObj<mtGC> {
   // previous and current epochs
   double mortality_rate(size_t prev_pop, size_t cur_pop);
 
+  // Update to a new epoch, creating a slot for new census.
+  void prepare_for_census_update();
+
   // Update the tenuring threshold, calling
-  // compute_tenuring_threshold to calculate the new
+  // compute_tenuring_threshold() to calculate the new
   // value
   void update_tenuring_threshold();
 
-  // This uses the data in the ShenandoahAgeCensus object's _global_age_table and the
-  // current _epoch to compute a new tenuring threshold, which will be remembered
-  // until the next invocation of compute_tenuring_threshold.
+  // Use _global_age_table and the current _epoch to compute a new tenuring
+  // threshold, which will be remembered until the next invocation of
+  // compute_tenuring_threshold.
   uint compute_tenuring_threshold();
 
+  // Return the tenuring threshold computed for the previous epoch
+  uint previous_tenuring_threshold() const {
+    assert(_epoch < MAX_SNAPSHOTS, "Error");
+    uint prev = _epoch - 1;
+    if (prev >= MAX_SNAPSHOTS) {
+      // _epoch is 0
+      assert(_epoch == 0, "Error");
+      prev = MAX_SNAPSHOTS - 1;
+    }
+    return _tenuring_threshold[prev];
+  }
+
+#ifndef PRODUCT
+  // Return the sum of size of objects of all ages recorded in the
+  // census at snapshot indexed by snap.
+  size_t get_all_ages(uint snap);
+
+  // Return the size of all objects that were encountered, but skipped,
+  // during the census, because their age was indeterminate.
+  size_t get_skipped(uint snap);
+
+  // Update the total size of objects counted or skipped at the census for
+  // the most recent epoch.
+  void update_total();
+#endif // !PRODUCT
+
  public:
   enum {
     MAX_COHORTS = AgeTable::table_size,    // = markWord::max_age + 1
@@ -151,37 +188,38 @@ class ShenandoahAgeCensus: public CHeapObj<mtGC> {
   void add_young(size_t size, uint worker_id);
 #endif // SHENANDOAH_CENSUS_NOISE
 
-  // Update to a new epoch, creating a slot for new census.
-  void prepare_for_census_update();
-
   // Update the census data, and compute the new tenuring threshold.
+  // This method should be called at the end of each marking (or optionally
+  // evacuation) cycle to update the tenuring threshold to be used in
+  // the next cycle.
   // age0_pop is the population of Cohort 0 that may have been missed in
-  // the regular census.
+  // the regular census during the marking cycle, corresponding to objects
+  // allocated when the concurrent marking was in progress.
+  // Optional parameters, pv1 and pv2 are population vectors that together
+  // provide object census data (only) for the case when
+  // ShenandoahGenerationalCensusAtEvac. In this case, the age0_pop
+  // is 0, because the evacuated objects have all had their ages incremented.
   void update_census(size_t age0_pop, AgeTable* pv1 = nullptr, AgeTable* pv2 = nullptr);
 
   // Return the most recently computed tenuring threshold
   uint tenuring_threshold() const { return _tenuring_threshold[_epoch]; }
 
-  // Return the tenuring threshold computed for the previous epoch
-  uint previous_tenuring_threshold() const {
-    assert(_epoch < MAX_SNAPSHOTS, "Error");
-    uint prev = _epoch - 1;
-    if (prev >= MAX_SNAPSHOTS) {
-      // _epoch is 0
-      prev = MAX_SNAPSHOTS - 1;
-    }
-    return _tenuring_threshold[prev];
-  }
-
   // Reset the epoch, clearing accumulated census history
+  // Note: this isn't currently used, but reserved for planned
+  // future usage.
   void reset_global();
-  // Reset any partial census information
+
+  // Reset any (potentially partial) census information in worker-local age tables
   void reset_local();
 
 #ifndef PRODUCT
   // Check whether census information is clear
   bool is_clear_global();
   bool is_clear_local();
+
+  // Return the net size of objects encountered (counted or skipped) in census
+  // at most recent epoch.
+  size_t get_total() { return _total; }
 #endif // !PRODUCT
 
   // Print the age census information

src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp

@@ -238,14 +238,12 @@ void ShenandoahConcurrentMark::concurrent_mark() {
   uint nworkers = workers->active_workers();
   task_queues()->reserve(nworkers);
 
+  ShenandoahGenerationType gen_type = _generation->type();
   ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set();
   ShenandoahFlushSATBHandshakeClosure flush_satb(qset);
   for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) {
-    switch (_generation->type()) {
+    switch (gen_type) {
       case YOUNG: {
-        // Clear any old/partial local census data before the start of marking.
-        heap->age_census()->reset_local();
-        assert(heap->age_census()->is_clear_local(), "Error");
         TaskTerminator terminator(nworkers, task_queues());
         ShenandoahConcurrentMarkingTask<YOUNG> task(this, &terminator);
         workers->run_task(&task);
@@ -258,9 +256,6 @@ void ShenandoahConcurrentMark::concurrent_mark() {
         break;
       }
       case GLOBAL: {
-        // Clear any old/partial local census data before the start of marking.
-        heap->age_census()->reset_local();
-        assert(heap->age_census()->is_clear_local(), "Error");
         TaskTerminator terminator(nworkers, task_queues());
         ShenandoahConcurrentMarkingTask<GLOBAL> task(this, &terminator);
         workers->run_task(&task);

src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp

@@ -149,14 +149,12 @@ ShenandoahCycleStats ShenandoahEvacuationTracker::flush_cycle_to_global() {
   _workers_global.accumulate(&workers);
 
   if (_generational && (ShenandoahGenerationalCensusAtEvac || !ShenandoahGenerationalAdaptiveTenuring)) {
-    // Ingest population vectors into the heap's global census
-    // data, and use it to compute an appropriate tenuring threshold
+    // Ingest mutator & worker collected population vectors into the heap's
+    // global census data, and use it to compute an appropriate tenuring threshold
     // for use in the next cycle.
-    ShenandoahAgeCensus* census = ShenandoahHeap::heap()->age_census();
-    census->prepare_for_census_update();
     // The first argument is used for any age 0 cohort population that we may otherwise have
     // missed during the census. This is non-zero only when census happens at marking.
-    census->update_census(0, _mutators_global.age_table(), _workers_global.age_table());
+    ShenandoahHeap::heap()->age_census()->update_census(0, _mutators_global.age_table(), _workers_global.age_table());
   }
 
   return {workers, mutators};

src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp

@@ -677,14 +677,20 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) {
     // young regions and sum the volume of objects between TAMS and top.
     ShenandoahUpdateCensusZeroCohortClosure age0_cl(complete_marking_context());
     heap->young_generation()->heap_region_iterate(&age0_cl);
-    size_t age0_pop = age0_cl.get_population();
+    size_t age0_pop = age0_cl.get_age0_population();
 
-    // Age table updates
-    ShenandoahAgeCensus* census = heap->age_census();
-    census->prepare_for_census_update();
     // Update the global census, including the missed age 0 cohort above,
-    // along with the census during marking, and compute the tenuring threshold
-    census->update_census(age0_pop);
+    // along with the census done during marking, and compute the tenuring threshold.
+    heap->age_census()->update_census(age0_pop);
+#ifndef PRODUCT
+    size_t total_pop = age0_cl.get_total_population();
+    size_t total_census = heap->age_census()->get_total();
+    // Usually total_pop > total_census, but not by too much.
+    // We use integer division so anything up to just less than 2 is considered
+    // reasonable, and the "+1" is to avoid divide-by-zero.
+    assert((total_pop+1)/(total_census+1) ==  1, "Extreme divergence: "
+           SIZE_FORMAT "/" SIZE_FORMAT, total_pop, total_census);
+#endif
   }
 
   {

src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp

@@ -122,3 +122,16 @@ void ShenandoahGlobalGeneration::set_mark_incomplete() {
   heap->young_generation()->set_mark_incomplete();
   heap->old_generation()->set_mark_incomplete();
 }
+
+void ShenandoahGlobalGeneration::prepare_gc() {
+  ShenandoahGeneration::prepare_gc();
+
+  if (ShenandoahHeap::heap()->mode()->is_generational()) {
+    assert(type() == GLOBAL, "Unexpected generation type");
+    // Clear any stale/partial local census data before the start of a
+    // new marking cycle
+    ShenandoahHeap::heap()->age_census()->reset_local();
+  } else {
+    assert(type() == NON_GEN, "Unexpected generation type");
+  }
+}

src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp

@@ -65,6 +65,8 @@ class ShenandoahGlobalGeneration : public ShenandoahGeneration {
   void set_mark_incomplete() override;
 
   ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override;
+
+  virtual void prepare_gc() override;
 };
 
 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGLOBALGENERATION_HPP

src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp

@@ -81,15 +81,18 @@ void ShenandoahFinalMarkUpdateRegionStateClosure::heap_region_do(ShenandoahHeapR
 
 ShenandoahUpdateCensusZeroCohortClosure::ShenandoahUpdateCensusZeroCohortClosure(
   ShenandoahMarkingContext *ctx) :
-  _ctx(ctx), _pop(0) {}
+  _ctx(ctx), _age0_pop(0), _total_pop(0) {}
 
 void ShenandoahUpdateCensusZeroCohortClosure::heap_region_do(ShenandoahHeapRegion* r) {
   if (_ctx != nullptr && r->is_active()) {
     assert(r->is_young(), "Young regions only");
     HeapWord* tams = _ctx->top_at_mark_start(r);
     HeapWord* top  = r->top();
     if (top > tams) {
-      _pop += pointer_delta(top, tams);
+      _age0_pop += pointer_delta(top, tams);
     }
+    // TODO: check significance of _ctx != nullptr above, can that
+    // spoof _total_pop in some corner cases?
+    NOT_PRODUCT(_total_pop += r->get_live_data_words();)
   }
 }

src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp

@@ -45,15 +45,20 @@ class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionC
 
 // Add [TAMS, top) volume over young regions. Used to correct age 0 cohort census
 // for adaptive tenuring when census is taken during marking.
+// In non-product builds, for the purposes of verification, we also collect the total
+// live objects in young regions as well.
 class ShenandoahUpdateCensusZeroCohortClosure : public ShenandoahHeapRegionClosure {
 private:
   ShenandoahMarkingContext* const _ctx;
-  size_t _pop;   // running tally of population
+  // Population size units are words (not bytes)
+  size_t _age0_pop;                // running tally of age0 population size
+  size_t _total_pop;               // total live population size
 public:
   ShenandoahUpdateCensusZeroCohortClosure(ShenandoahMarkingContext* ctx);
 
   void heap_region_do(ShenandoahHeapRegion* r);
 
-  size_t get_population() { return _pop; }
+  size_t get_age0_population()  { return _age0_pop; }
+  size_t get_total_population() { return _total_pop; }
 };
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHMARKCLOSURES_HPP

src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp

@@ -103,3 +103,12 @@ size_t ShenandoahYoungGeneration::soft_available() const {
   return MIN2(available, ShenandoahHeap::heap()->free_set()->available());
 }
 
+void ShenandoahYoungGeneration::prepare_gc() {
+
+  ShenandoahGeneration::prepare_gc();
+
+  assert(type() == YOUNG, "Error?");
+  // Clear any stale/partial local census data before the start of a
+  // new marking cycle
+  ShenandoahHeap::heap()->age_census()->reset_local();
+}

src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp

@@ -77,6 +77,8 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration {
   // Do not override available_with_reserve() because that needs to see memory reserved for Collector
 
   size_t soft_available() const override;
+
+  virtual void prepare_gc() override;
 };
 
 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHYOUNGGENERATION_HPP