Account for humongous object waste

Reviewed-by: ysr, kdnilsen

Author: William Kemper

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp

@@ -579,13 +579,23 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) {
                         PTR_FORMAT " at transition from FREE to %s",
                         p2i(r->bottom()), p2i(r->end()), p2i(ctx->top_bitmap(r)), req.affiliation_name());
 
+    // While individual regions report their true use, all humongous regions are marked used in the free set.
     _mutator_free_bitmap.clear_bit(r->index());
   }
-
-  // While individual regions report their true use, all humongous regions are
-  // marked used in the free set.
-  increase_used(ShenandoahHeapRegion::region_size_bytes() * num);
-  generation->increase_used(words_size * HeapWordSize);
+  size_t total_humongous_size = ShenandoahHeapRegion::region_size_bytes() * num;
+  increase_used(total_humongous_size);
+  if (_heap->mode()->is_generational()) {
+    size_t humongous_waste = total_humongous_size - words_size * HeapWordSize;
+    _heap->global_generation()->increase_used(words_size * HeapWordSize);
+    _heap->global_generation()->increase_humongous_waste(humongous_waste);
+    if (req.is_young()) {
+      _heap->young_generation()->increase_used(words_size * HeapWordSize);
+      _heap->young_generation()->increase_humongous_waste(humongous_waste);
+    } else if (req.is_old()) {
+      _heap->old_generation()->increase_used(words_size * HeapWordSize);
+      _heap->old_generation()->increase_humongous_waste(humongous_waste);
+    }
+  }
 
   if (remainder != 0) {
     // Record this remainder as allocation waste

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp

@@ -68,6 +68,7 @@ class ShenandoahFreeSet : public CHeapObj<mtGC> {
   void adjust_bounds();
   bool touches_bounds(size_t num) const;
 
+  // Used of free set represents the amount of is_mutator_free set that has been consumed since most recent rebuild.
   void increase_used(size_t amount);
   void clear_internal();
 

src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp

@@ -202,8 +202,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
     // Defer unadjust_available() invocations until after Full GC finishes its efforts because Full GC makes use
     // of young-gen memory that may have been loaned from old-gen.
 
-    // No need to old_gen->increase_used().  That was done when plabs were allocated, accounting for both old evacs and promotions.
-
+    // No need for old_gen->increase_used() as this was done when plabs were allocated.
     heap->set_alloc_supplement_reserve(0);
     heap->set_young_evac_reserve(0);
     heap->set_old_evac_reserve(0);
@@ -1502,10 +1501,9 @@ void ShenandoahFullGC::phase5_epilog() {
                    byte_size_in_proper_unit(heap->young_generation()->used()), proper_unit_for_byte_size(heap->young_generation()->used()),
                    byte_size_in_proper_unit(heap->old_generation()->used()),   proper_unit_for_byte_size(heap->old_generation()->used()));
     }
-
     heap->collection_set()->clear();
     heap->free_set()->rebuild();
   }
-
   heap->clear_cancelled_gc(true /* clear oom handler */);
 }
+

src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp

@@ -162,17 +162,16 @@ void ShenandoahGeneration::log_status(const char *msg) const {
   size_t v_soft_max_capacity = soft_max_capacity();
   size_t v_max_capacity = max_capacity();
   size_t v_available = available();
-  size_t v_adjusted_avail = adjusted_available();
+  size_t v_humongous_waste = get_humongous_waste();
   LogGcInfo::print("%s: %s generation used: " SIZE_FORMAT "%s, used regions: " SIZE_FORMAT "%s, "
-                   "soft capacity: " SIZE_FORMAT "%s, max capacity: " SIZE_FORMAT "%s, available: " SIZE_FORMAT "%s, "
-                   "adjusted available: " SIZE_FORMAT "%s",
-                   msg, name(),
+                   "humongous waste: " SIZE_FORMAT "%s, soft capacity: " SIZE_FORMAT "%s, max capacity: " SIZE_FORMAT "%s, "
+                   "available: " SIZE_FORMAT "%s", msg, name(),
                    byte_size_in_proper_unit(v_used),              proper_unit_for_byte_size(v_used),
                    byte_size_in_proper_unit(v_used_regions),      proper_unit_for_byte_size(v_used_regions),
+                   byte_size_in_proper_unit(v_humongous_waste),   proper_unit_for_byte_size(v_humongous_waste),
                    byte_size_in_proper_unit(v_soft_max_capacity), proper_unit_for_byte_size(v_soft_max_capacity),
                    byte_size_in_proper_unit(v_max_capacity),      proper_unit_for_byte_size(v_max_capacity),
-                   byte_size_in_proper_unit(v_available),         proper_unit_for_byte_size(v_available),
-                   byte_size_in_proper_unit(v_adjusted_avail),    proper_unit_for_byte_size(v_adjusted_avail));
+                   byte_size_in_proper_unit(v_available),         proper_unit_for_byte_size(v_available));
 }
 
 void ShenandoahGeneration::reset_mark_bitmap() {
@@ -893,7 +892,7 @@ ShenandoahGeneration::ShenandoahGeneration(ShenandoahGenerationType type,
   _task_queues(new ShenandoahObjToScanQueueSet(max_workers)),
   _ref_processor(new ShenandoahReferenceProcessor(MAX2(max_workers, 1U))),
   _collection_thread_time_s(0.0),
-  _affiliated_region_count(0), _used(0), _bytes_allocated_since_gc_start(0),
+  _affiliated_region_count(0), _humongous_waste(0), _used(0), _bytes_allocated_since_gc_start(0),
   _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity),
   _adjusted_capacity(soft_max_capacity), _heuristics(nullptr) {
   _is_marking_complete.set();
@@ -957,13 +956,15 @@ size_t ShenandoahGeneration::decrement_affiliated_region_count() {
 }
 
 void ShenandoahGeneration::establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste) {
+  assert(ShenandoahHeap::heap()->mode()->is_generational(), "Only generational mode accounts for generational usage");
   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
   _affiliated_region_count = num_regions;
   _used = num_bytes;
-  // future improvement: _humongous_waste = humongous_waste;
+  _humongous_waste = humongous_waste;
 }
 
 void ShenandoahGeneration::clear_used() {
+  assert(ShenandoahHeap::heap()->mode()->is_generational(), "Only generational mode accounts for generational usage");
   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
   // Do this atomically to assure visibility to other threads, even though these other threads may be idle "right now"..
   Atomic::store(&_used, (size_t)0);
@@ -978,6 +979,23 @@ void ShenandoahGeneration::decrease_used(size_t bytes) {
   Atomic::sub(&_used, bytes);
 }
 
+void ShenandoahGeneration::increase_humongous_waste(size_t bytes) {
+  if (bytes > 0) {
+    shenandoah_assert_heaplocked_or_fullgc_safepoint();
+    _humongous_waste += bytes;
+  }
+}
+
+void ShenandoahGeneration::decrease_humongous_waste(size_t bytes) {
+  if (bytes > 0) {
+    shenandoah_assert_heaplocked_or_fullgc_safepoint();
+    assert(_humongous_waste >= bytes, "Waste cannot be negative");
+    assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || (_humongous_waste >= bytes),
+           "Waste (" SIZE_FORMAT ") cannot be negative (after subtracting " SIZE_FORMAT ")", _humongous_waste, bytes);
+    _humongous_waste -= bytes;
+  }
+}
+
 size_t ShenandoahGeneration::used_regions() const {
   return _affiliated_region_count;
 }
@@ -997,19 +1015,14 @@ size_t ShenandoahGeneration::used_regions_size() const {
 }
 
 size_t ShenandoahGeneration::available() const {
-  size_t in_use = used();
+  size_t in_use = used() + get_humongous_waste();
   size_t soft_capacity = soft_max_capacity();
   return in_use > soft_capacity ? 0 : soft_capacity - in_use;
 }
 
 size_t ShenandoahGeneration::adjust_available(intptr_t adjustment) {
-  // TODO: ysr: remove this check & warning
-  if (adjustment % ShenandoahHeapRegion::region_size_bytes() != 0) {
-    log_warning(gc)("Adjustment (" INTPTR_FORMAT ") should be a multiple of region size (" SIZE_FORMAT ")",
-                    adjustment, ShenandoahHeapRegion::region_size_bytes());
-  }
   assert(adjustment % ShenandoahHeapRegion::region_size_bytes() == 0,
-         "Adjustment to generation size must be multiple of region size");
+        "Adjustment to generation size must be multiple of region size");
   _adjusted_capacity = soft_max_capacity() + adjustment;
   return _adjusted_capacity;
 }
@@ -1020,7 +1033,7 @@ size_t ShenandoahGeneration::unadjust_available() {
 }
 
 size_t ShenandoahGeneration::adjusted_available() const {
-  size_t in_use = used();
+  size_t in_use = used() + get_humongous_waste();
   size_t capacity = _adjusted_capacity;
   return in_use > capacity ? 0 : capacity - in_use;
 }

src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp

@@ -52,6 +52,14 @@ class ShenandoahGeneration : public CHeapObj<mtGC> {
 protected:
   // Usage
   size_t _affiliated_region_count;
+
+  // How much free memory is left in the last region of humongous objects.
+  // This is _not_ included in used, but it _is_ deducted from available,
+  // which gives the heuristics a more accurate view of how much memory remains
+  // for allocation. This figure is also included the heap status logging.
+  // The units are bytes. The value is only changed on a safepoint or under the
+  // heap lock.
+  size_t _humongous_waste;
   volatile size_t _used;
   volatile size_t _bytes_allocated_since_gc_start;
   size_t _max_capacity;
@@ -196,6 +204,10 @@ class ShenandoahGeneration : public CHeapObj<mtGC> {
   void increase_used(size_t bytes);
   void decrease_used(size_t bytes);
 
+  void increase_humongous_waste(size_t bytes);
+  void decrease_humongous_waste(size_t bytes);
+  size_t get_humongous_waste() const { return _humongous_waste; }
+
   virtual bool is_concurrent_mark_in_progress() = 0;
   void confirm_heuristics_mode();
 

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp

@@ -282,11 +282,17 @@ void ShenandoahHeapRegion::make_trash() {
   shenandoah_assert_heaplocked();
   reset_age();
   switch (_state) {
-    case _cset:
-      // Reclaiming cset regions
     case _humongous_start:
     case _humongous_cont:
-      // Reclaiming humongous regions
+    {
+      // Reclaiming humongous regions and reclaim humongous waste.  When this region is eventually recycled, we'll reclaim
+      // its used memory.  At recycle time, we no longer recognize this as a humongous region.
+      if (ShenandoahHeap::heap()->mode()->is_generational()) {
+        decrement_humongous_waste();
+      }
+    }
+    case _cset:
+      // Reclaiming cset regions
     case _regular:
       // Immediate region reclaim
       set_state(_trash);
@@ -655,7 +661,9 @@ void ShenandoahHeapRegion::recycle() {
   ShenandoahHeap* heap = ShenandoahHeap::heap();
   shenandoah_assert_heaplocked();
 
-  heap->generation_for(affiliation())->decrease_used(used());
+  if (ShenandoahHeap::heap()->mode()->is_generational()) {
+    heap->generation_for(affiliation())->decrease_used(used());
+  }
 
   set_top(bottom());
   clear_live_data();
@@ -1036,9 +1044,16 @@ size_t ShenandoahHeapRegion::promote_humongous() {
         log_debug(gc)("promoting humongous region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT,
                       r->index(), p2i(r->bottom()), p2i(r->top()));
         // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here
-        r->set_affiliation(OLD_GENERATION);
         old_generation->increase_used(r->used());
         young_generation->decrease_used(r->used());
+        r->set_affiliation(OLD_GENERATION);
+      }
+
+      ShenandoahHeapRegion* tail = heap->get_region(index_limit - 1);
+      size_t waste = tail->free();
+      if (waste != 0) {
+        old_generation->increase_humongous_waste(waste);
+        young_generation->decrease_humongous_waste(waste);
       }
       // Then fall through to finish the promotion after releasing the heap lock.
     } else {
@@ -1073,3 +1088,12 @@ size_t ShenandoahHeapRegion::promote_humongous() {
   }
   return index_limit - index();
 }
+
+void ShenandoahHeapRegion::decrement_humongous_waste() const {
+  assert(is_humongous(), "Should only use this for humongous regions");
+  size_t waste_bytes = free();
+  if (waste_bytes > 0) {
+    ShenandoahHeap::heap()->generation_for(affiliation())->decrease_humongous_waste(waste_bytes);
+    ShenandoahHeap::heap()->global_generation()->decrease_humongous_waste(waste_bytes);
+  }
+}

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -461,6 +461,7 @@ class ShenandoahHeapRegion {
   size_t promote_humongous();
 
 private:
+  void decrement_humongous_waste() const;
   void do_commit();
   void do_uncommit();
 

src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp

@@ -347,14 +347,17 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
 // a subset (e.g. the young generation or old generation) of the total heap.
 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure {
 private:
-  size_t _used, _committed, _garbage, _regions;
+  size_t _used, _committed, _garbage, _regions, _humongous_waste;
 public:
-  ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0), _regions(0) {};
+  ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0), _regions(0), _humongous_waste(0) {};
 
   void heap_region_do(ShenandoahHeapRegion* r) {
     _used += r->used();
     _garbage += r->garbage();
     _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0;
+    if (r->is_humongous()) {
+      _humongous_waste += r->free();
+    }
     _regions++;
     log_debug(gc)("ShenandoahCalculateRegionStatsClosure: adding " SIZE_FORMAT " for %s Region " SIZE_FORMAT ", yielding: " SIZE_FORMAT,
             r->used(), (r->is_humongous() ? "humongous" : "regular"), r->index(), _used);
@@ -364,6 +367,7 @@ class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure
   size_t committed() { return _committed; }
   size_t garbage() { return _garbage; }
   size_t regions() { return _regions; }
+  size_t waste() { return _humongous_waste; }
 
   // span is the total memory affiliated with these stats (some of which is in use and other is available)
   size_t span() { return _regions * ShenandoahHeapRegion::region_size_bytes(); }
@@ -402,7 +406,7 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure {
   static void validate_usage(const char* label, ShenandoahGeneration* generation, ShenandoahCalculateRegionStatsClosure& stats) {
     size_t generation_used = generation->used();
     guarantee(stats.used() == generation_used,
-              "%s: generation (%s) used size must be consistent: generation-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s",
+              "%s: generation (%s) used size must be consistent: generation-used: " SIZE_FORMAT "%s, regions-used: " SIZE_FORMAT "%s",
               label, generation->name(),
               byte_size_in_proper_unit(generation_used), proper_unit_for_byte_size(generation_used),
               byte_size_in_proper_unit(stats.used()),    proper_unit_for_byte_size(stats.used()));
@@ -418,6 +422,12 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure {
 //              label, generation->name(), stats.regions(),
 //              byte_size_in_proper_unit(capacity), proper_unit_for_byte_size(capacity));
 
+    size_t humongous_waste = generation->get_humongous_waste();
+    guarantee(stats.waste() == humongous_waste,
+              "%s: generation (%s) humongous waste must be consistent: generation: " SIZE_FORMAT "%s, regions: " SIZE_FORMAT "%s",
+              label, generation->name(),
+              byte_size_in_proper_unit(humongous_waste), proper_unit_for_byte_size(humongous_waste),
+              byte_size_in_proper_unit(stats.waste()),   proper_unit_for_byte_size(stats.waste()));
   }
 };