8191565: Last-ditch Full GC should also move humongous objects

Reviewed-by: tschatzl, sjohanss

Author: Ivan Walulya

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -193,80 +193,56 @@ HeapRegion* G1CollectedHeap::new_region(size_t word_size,
   return res;
 }
 
-HeapWord*
-G1CollectedHeap::humongous_obj_allocate_initialize_regions(HeapRegion* first_hr,
-                                                           uint num_regions,
-                                                           size_t word_size) {
-  assert(first_hr != NULL, "pre-condition");
-  assert(is_humongous(word_size), "word_size should be humongous");
-  assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition");
-
-  // Index of last region in the series.
-  uint first = first_hr->hrm_index();
-  uint last = first + num_regions - 1;
-
-  // We need to initialize the region(s) we just discovered. This is
-  // a bit tricky given that it can happen concurrently with
-  // refinement threads refining cards on these regions and
-  // potentially wanting to refine the BOT as they are scanning
-  // those cards (this can happen shortly after a cleanup; see CR
-  // 6991377). So we have to set up the region(s) carefully and in
-  // a specific order.
-
-  // The word size sum of all the regions we will allocate.
-  size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords;
+void G1CollectedHeap::set_humongous_metadata(HeapRegion* first_hr,
+                                             uint num_regions,
+                                             size_t word_size,
+                                             bool update_remsets) {
+  // Calculate the new top of the humongous object.
+  HeapWord* obj_top = first_hr->bottom() + word_size;
+  // The word size sum of all the regions used
+  size_t word_size_sum = num_regions * HeapRegion::GrainWords;
   assert(word_size <= word_size_sum, "sanity");
 
-  // The passed in hr will be the "starts humongous" region. The header
-  // of the new object will be placed at the bottom of this region.
-  HeapWord* new_obj = first_hr->bottom();
-  // This will be the new top of the new object.
-  HeapWord* obj_top = new_obj + word_size;
-
-  // First, we need to zero the header of the space that we will be
-  // allocating. When we update top further down, some refinement
-  // threads might try to scan the region. By zeroing the header we
-  // ensure that any thread that will try to scan the region will
-  // come across the zero klass word and bail out.
-  //
-  // NOTE: It would not have been correct to have used
-  // CollectedHeap::fill_with_object() and make the space look like
-  // an int array. The thread that is doing the allocation will
-  // later update the object header to a potentially different array
-  // type and, for a very short period of time, the klass and length
-  // fields will be inconsistent. This could cause a refinement
-  // thread to calculate the object size incorrectly.
-  Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);
+  // How many words memory we "waste" which cannot hold a filler object.
+  size_t words_not_fillable = 0;
 
-  // Next, pad out the unused tail of the last region with filler
+  // Pad out the unused tail of the last region with filler
   // objects, for improved usage accounting.
-  // How many words we use for filler objects.
-  size_t word_fill_size = word_size_sum - word_size;
 
-  // How many words memory we "waste" which cannot hold a filler object.
-  size_t words_not_fillable = 0;
+  // How many words can we use for filler objects.
+  size_t words_fillable = word_size_sum - word_size;
 
-  if (word_fill_size >= min_fill_size()) {
-    fill_with_objects(obj_top, word_fill_size);
-  } else if (word_fill_size > 0) {
+  if (words_fillable >= G1CollectedHeap::min_fill_size()) {
+    G1CollectedHeap::fill_with_objects(obj_top, words_fillable);
+  } else {
     // We have space to fill, but we cannot fit an object there.
-    words_not_fillable = word_fill_size;
-    word_fill_size = 0;
+    words_not_fillable = words_fillable;
+    words_fillable = 0;
   }
 
   // We will set up the first region as "starts humongous". This
   // will also update the BOT covering all the regions to reflect
   // that there is a single object that starts at the bottom of the
   // first region.
-  first_hr->set_starts_humongous(obj_top, word_fill_size);
-  _policy->remset_tracker()->update_at_allocate(first_hr);
-  // Then, if there are any, we will set up the "continues
-  // humongous" regions.
-  HeapRegion* hr = NULL;
+  first_hr->hr_clear(false /* clear_space */);
+  first_hr->set_starts_humongous(obj_top, words_fillable);
+
+  if (update_remsets) {
+    _policy->remset_tracker()->update_at_allocate(first_hr);
+  }
+
+  // Indices of first and last regions in the series.
+  uint first = first_hr->hrm_index();
+  uint last = first + num_regions - 1;
+
+  HeapRegion* hr = nullptr;
   for (uint i = first + 1; i <= last; ++i) {
     hr = region_at(i);
+    hr->hr_clear(false /* clear_space */);
     hr->set_continues_humongous(first_hr);
-    _policy->remset_tracker()->update_at_allocate(hr);
+    if (update_remsets) {
+      _policy->remset_tracker()->update_at_allocate(hr);
+    }
   }
 
   // Up to this point no concurrent thread would have been able to
@@ -297,11 +273,57 @@ G1CollectedHeap::humongous_obj_allocate_initialize_regions(HeapRegion* first_hr,
   assert(words_not_fillable == 0 ||
          first_hr->bottom() + word_size_sum - words_not_fillable == hr->top(),
          "Miscalculation in humongous allocation");
+}
+
+HeapWord*
+G1CollectedHeap::humongous_obj_allocate_initialize_regions(HeapRegion* first_hr,
+                                                           uint num_regions,
+                                                           size_t word_size) {
+  assert(first_hr != NULL, "pre-condition");
+  assert(is_humongous(word_size), "word_size should be humongous");
+  assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition");
+
+  // Index of last region in the series.
+  uint first = first_hr->hrm_index();
+  uint last = first + num_regions - 1;
+
+  // We need to initialize the region(s) we just discovered. This is
+  // a bit tricky given that it can happen concurrently with
+  // refinement threads refining cards on these regions and
+  // potentially wanting to refine the BOT as they are scanning
+  // those cards (this can happen shortly after a cleanup; see CR
+  // 6991377). So we have to set up the region(s) carefully and in
+  // a specific order.
 
-  increase_used((word_size_sum - words_not_fillable) * HeapWordSize);
+  // The passed in hr will be the "starts humongous" region. The header
+  // of the new object will be placed at the bottom of this region.
+  HeapWord* new_obj = first_hr->bottom();
+
+  // First, we need to zero the header of the space that we will be
+  // allocating. When we update top further down, some refinement
+  // threads might try to scan the region. By zeroing the header we
+  // ensure that any thread that will try to scan the region will
+  // come across the zero klass word and bail out.
+  //
+  // NOTE: It would not have been correct to have used
+  // CollectedHeap::fill_with_object() and make the space look like
+  // an int array. The thread that is doing the allocation will
+  // later update the object header to a potentially different array
+  // type and, for a very short period of time, the klass and length
+  // fields will be inconsistent. This could cause a refinement
+  // thread to calculate the object size incorrectly.
+  Copy::fill_to_words(new_obj, oopDesc::header_size(), 0);
+
+  // Next, update the metadata for the regions.
+  set_humongous_metadata(first_hr, num_regions, word_size, true);
+
+  HeapRegion* last_hr = region_at(last);
+  size_t used = byte_size(first_hr->bottom(), last_hr->top());
+
+  increase_used(used);
 
   for (uint i = first; i <= last; ++i) {
-    hr = region_at(i);
+    HeapRegion *hr = region_at(i);
     _humongous_set.add(hr);
     _hr_printer.alloc(hr);
   }

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -606,6 +606,11 @@ class G1CollectedHeap : public CollectedHeap {
   // Register the given region to be part of the collection set.
   inline void register_humongous_candidate_region_with_region_attr(uint index);
 
+  void set_humongous_metadata(HeapRegion* first_hr,
+                              uint num_regions,
+                              size_t word_size,
+                              bool update_remsets);
+
   // We register a region with the fast "in collection set" test. We
   // simply set to true the array slot corresponding to this region.
   void register_young_region_with_region_attr(HeapRegion* r) {

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -40,7 +40,7 @@
 #include "gc/g1/g1Policy.hpp"
 #include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
-#include "gc/shared/preservedMarks.hpp"
+#include "gc/shared/preservedMarks.inline.hpp"
 #include "gc/shared/referenceProcessor.hpp"
 #include "gc/shared/verifyOption.hpp"
 #include "gc/shared/weakProcessor.inline.hpp"
@@ -119,12 +119,15 @@ G1FullCollector::G1FullCollector(G1CollectedHeap* heap,
     _scope(heap->monitoring_support(), explicit_gc, clear_soft_refs, do_maximal_compaction, tracer),
     _num_workers(calc_active_workers()),
     _has_compaction_targets(false),
+    _has_humongous(false),
     _oop_queue_set(_num_workers),
     _array_queue_set(_num_workers),
     _preserved_marks_set(true),
     _serial_compaction_point(this),
+    _humongous_compaction_point(this),
     _is_alive(this, heap->concurrent_mark()->mark_bitmap()),
     _is_alive_mutator(heap->ref_processor_stw(), &_is_alive),
+    _humongous_compaction_regions(8),
     _always_subject_to_discovery(),
     _is_subject_mutator(heap->ref_processor_stw(), &_always_subject_to_discovery),
     _region_attr_table() {
@@ -155,6 +158,7 @@ G1FullCollector::~G1FullCollector() {
     delete _markers[i];
     delete _compaction_points[i];
   }
+
   FREE_C_HEAP_ARRAY(G1FullGCMarker*, _markers);
   FREE_C_HEAP_ARRAY(G1FullGCCompactionPoint*, _compaction_points);
   FREE_C_HEAP_ARRAY(HeapWord*, _compaction_tops);
@@ -246,6 +250,8 @@ void G1FullCollector::complete_collection() {
   _heap->gc_epilogue(true);
 
   _heap->verify_after_full_collection();
+
+  _heap->print_heap_after_full_collection();
 }
 
 void G1FullCollector::before_marking_update_attribute_table(HeapRegion* hr) {
@@ -343,6 +349,12 @@ void G1FullCollector::phase2_prepare_compaction() {
   // maximally compact the tail regions of the compaction queues serially.
   if (scope()->do_maximal_compaction() || !has_free_compaction_targets) {
     phase2c_prepare_serial_compaction();
+
+    if (scope()->do_maximal_compaction() &&
+        has_humongous() &&
+        serial_compaction_point()->has_regions()) {
+      phase2d_prepare_humongous_compaction();
+    }
   }
 }
 
@@ -418,6 +430,35 @@ void G1FullCollector::phase2c_prepare_serial_compaction() {
   serial_cp->update();
 }
 
+void G1FullCollector::phase2d_prepare_humongous_compaction() {
+  GCTraceTime(Debug, gc, phases) debug("Phase 2: Prepare humongous compaction", scope()->timer());
+  G1FullGCCompactionPoint* serial_cp = serial_compaction_point();
+  assert(serial_cp->has_regions(), "Sanity!" );
+
+  uint last_serial_target = serial_cp->current_region()->hrm_index();
+  uint region_index = last_serial_target + 1;
+  uint max_reserved_regions = _heap->max_reserved_regions();
+
+  G1FullGCCompactionPoint* humongous_cp = humongous_compaction_point();
+
+  while (region_index < max_reserved_regions) {
+    HeapRegion* hr = _heap->region_at_or_null(region_index);
+
+    if (hr == nullptr) {
+      region_index++;
+      continue;
+    } else if (hr->is_starts_humongous()) {
+      uint num_regions = humongous_cp->forward_humongous(hr);
+      region_index += num_regions; // Skip over the continues humongous regions.
+      continue;
+    } else if (is_compaction_target(region_index)) {
+      // Add the region to the humongous compaction point.
+      humongous_cp->add(hr);
+    }
+    region_index++;
+  }
+}
+
 void G1FullCollector::phase3_adjust_pointers() {
   // Adjust the pointers to reflect the new locations
   GCTraceTime(Info, gc, phases) info("Phase 3: Adjust pointers", scope()->timer());
@@ -436,6 +477,11 @@ void G1FullCollector::phase4_do_compaction() {
   if (serial_compaction_point()->has_regions()) {
     task.serial_compaction();
   }
+
+  if (!_humongous_compaction_regions.is_empty()) {
+    assert(scope()->do_maximal_compaction(), "Only compact humongous during maximal compaction");
+    task.humongous_compaction();
+  }
 }
 
 void G1FullCollector::phase5_reset_metadata() {

src/hotspot/share/gc/g1/g1FullCollector.hpp

@@ -76,15 +76,18 @@ class G1FullCollector : StackObj {
   G1FullGCScope             _scope;
   uint                      _num_workers;
   bool                      _has_compaction_targets;
+  bool                      _has_humongous;
   G1FullGCMarker**          _markers;
   G1FullGCCompactionPoint** _compaction_points;
   OopQueueSet               _oop_queue_set;
   ObjArrayTaskQueueSet      _array_queue_set;
   PreservedMarksSet         _preserved_marks_set;
   G1FullGCCompactionPoint   _serial_compaction_point;
+  G1FullGCCompactionPoint   _humongous_compaction_point;
   G1IsAliveClosure          _is_alive;
   ReferenceProcessorIsAliveMutator _is_alive_mutator;
   G1RegionMarkStats*        _live_stats;
+  GrowableArrayCHeap<HeapRegion*, mtGC> _humongous_compaction_regions;
 
   static uint calc_active_workers();
 
@@ -115,6 +118,7 @@ class G1FullCollector : StackObj {
   ObjArrayTaskQueueSet*    array_queue_set() { return &_array_queue_set; }
   PreservedMarksSet*       preserved_mark_set() { return &_preserved_marks_set; }
   G1FullGCCompactionPoint* serial_compaction_point() { return &_serial_compaction_point; }
+  G1FullGCCompactionPoint* humongous_compaction_point() { return &_humongous_compaction_point; }
   G1CMBitMap*              mark_bitmap();
   ReferenceProcessor*      reference_processor();
   size_t live_words(uint region_index) const {
@@ -134,22 +138,30 @@ class G1FullCollector : StackObj {
   inline void set_free(uint region_idx);
   inline bool is_free(uint region_idx) const;
   inline void update_from_compacting_to_skip_compacting(uint region_idx);
+  inline void update_from_skip_compacting_to_compacting(uint region_idx);
 
   inline void set_compaction_top(HeapRegion* r, HeapWord* value);
   inline HeapWord* compaction_top(HeapRegion* r) const;
 
   inline void set_has_compaction_targets();
   inline bool has_compaction_targets() const;
 
+  inline void add_humongous_region(HeapRegion* hr);
+  inline GrowableArrayCHeap<HeapRegion*, mtGC>& humongous_compaction_regions();
+
   uint truncate_parallel_cps();
 
+  inline void set_has_humongous();
+  inline bool has_humongous();
+
 private:
   void phase1_mark_live_objects();
   void phase2_prepare_compaction();
 
   void phase2a_determine_worklists();
   bool phase2b_forward_oops();
   void phase2c_prepare_serial_compaction();
+  void phase2d_prepare_humongous_compaction();
 
   void phase3_adjust_pointers();
   void phase4_do_compaction();

src/hotspot/share/gc/g1/g1FullCollector.inline.hpp

@@ -61,6 +61,11 @@ void G1FullCollector::update_from_compacting_to_skip_compacting(uint region_idx)
   _region_attr_table.set_skip_compacting(region_idx);
 }
 
+void G1FullCollector::update_from_skip_compacting_to_compacting(uint region_idx) {
+  DEBUG_ONLY(_region_attr_table.verify_is_skip_compacting(region_idx);)
+  _region_attr_table.set_compacting(region_idx);
+}
+
 void G1FullCollector::set_compaction_top(HeapRegion* r, HeapWord* value) {
   Atomic::store(&_compaction_tops[r->hrm_index()], value);
 }
@@ -79,5 +84,23 @@ bool G1FullCollector::has_compaction_targets() const {
   return _has_compaction_targets;
 }
 
+void G1FullCollector::set_has_humongous() {
+  if (!_has_humongous) {
+    _has_humongous = true;
+  }
+}
+
+bool G1FullCollector::has_humongous() {
+  return _has_humongous;
+}
+
+void G1FullCollector::add_humongous_region(HeapRegion* hr) {
+  _humongous_compaction_regions.append(hr);
+}
+
+GrowableArrayCHeap<HeapRegion*, mtGC>& G1FullCollector::humongous_compaction_regions() {
+  return _humongous_compaction_regions;
+}
+
 #endif // SHARE_GC_G1_G1FULLCOLLECTOR_INLINE_HPP