8314935: Shenandoah: Unable to throw OOME on back-to-back Full GCs

src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp

@@ -40,7 +40,8 @@ class ShenandoahCollectorPolicy : public CHeapObj<mtGC> {
 private:
   size_t _success_concurrent_gcs;
   size_t _success_degenerated_gcs;
-  size_t _success_full_gcs;
+  // Written by control thread, read by mutators
+  volatile size_t _success_full_gcs;
   size_t _alloc_failure_degenerated;
   size_t _alloc_failure_degenerated_upgrade_to_full;
   size_t _alloc_failure_full;
@@ -82,6 +83,10 @@ class ShenandoahCollectorPolicy : public CHeapObj<mtGC> {
   size_t cycle_counter() const;
 
   void print_gc_stats(outputStream* out) const;
+
+  size_t full_gc_count() const {
+    return _success_full_gcs + _alloc_failure_degenerated_upgrade_to_full;
+  }
 };
 
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCOLLECTORPOLICY_HPP

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -878,25 +878,14 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
     // It might happen that one of the threads requesting allocation would unblock
     // way later after GC happened, only to fail the second allocation, because
     // other threads have already depleted the free storage. In this case, a better
-    // strategy is to try again, as long as GC makes progress.
-    //
-    // Then, we need to make sure the allocation was retried after at least one
-    // Full GC, which means we want to try more than ShenandoahFullGCThreshold times.
-
-    size_t tries = 0;
-
-    while (result == nullptr && _progress_last_gc.is_set()) {
-      tries++;
-      control_thread()->handle_alloc_failure(req);
-      result = allocate_memory_under_lock(req, in_new_region);
-    }
-
-    while (result == nullptr && tries <= ShenandoahFullGCThreshold) {
-      tries++;
+    // strategy is to try again, as long as GC makes progress (or until at least
+    // one full GC has completed).
+    size_t original_count = shenandoah_policy()->full_gc_count();
+    while (result == nullptr
+        && (_progress_last_gc.is_set() || original_count == shenandoah_policy()->full_gc_count())) {
       control_thread()->handle_alloc_failure(req);
       result = allocate_memory_under_lock(req, in_new_region);
     }
-
   } else {
     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
     result = allocate_memory_under_lock(req, in_new_region);