8334594: Generational ZGC: Deadlock after OopMap rewrites in 8331572

src/hotspot/share/gc/shared/gcVMOperations.cpp

@@ -132,7 +132,7 @@ bool VM_GC_Operation::doit_prologue() {
 void VM_GC_Operation::doit_epilogue() {
   // GC thread root traversal likely used OopMapCache a lot, which
   // might have created lots of old entries. Trigger the cleanup now.
-  OopMapCache::trigger_cleanup();
+  OopMapCache::try_trigger_cleanup();
   if (Universe::has_reference_pending_list()) {
     Heap_lock->notify_all();
   }

src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp

@@ -44,7 +44,7 @@ void VM_ShenandoahOperation::doit_epilogue() {
   assert(!ShenandoahHeap::heap()->has_gc_state_changed(), "GC State was not synchronized to java threads.");
   // GC thread root traversal likely used OopMapCache a lot, which
   // might have created lots of old entries. Trigger the cleanup now.
-  OopMapCache::trigger_cleanup();
+  OopMapCache::try_trigger_cleanup();
 }
 
 bool VM_ShenandoahReferenceOperation::doit_prologue() {

src/hotspot/share/gc/x/xDriver.cpp

@@ -134,7 +134,7 @@ class VM_XOperation : public VM_Operation {
 
     // GC thread root traversal likely used OopMapCache a lot, which
     // might have created lots of old entries. Trigger the cleanup now.
-    OopMapCache::trigger_cleanup();
+    OopMapCache::try_trigger_cleanup();
   }
 
   bool gc_locked() const {

src/hotspot/share/gc/z/zGeneration.cpp

@@ -456,7 +456,7 @@ class VM_ZOperation : public VM_Operation {
 
     // GC thread root traversal likely used OopMapCache a lot, which
     // might have created lots of old entries. Trigger the cleanup now.
-    OopMapCache::trigger_cleanup();
+    OopMapCache::try_trigger_cleanup();
   }
 
   bool success() const {

src/hotspot/share/interpreter/oopMapCache.cpp

@@ -592,10 +592,13 @@ bool OopMapCache::has_cleanup_work() {
   return Atomic::load(&_old_entries) != nullptr;
 }
 
-void OopMapCache::trigger_cleanup() {
-  if (has_cleanup_work()) {
-    MutexLocker ml(Service_lock, Mutex::_no_safepoint_check_flag);
+void OopMapCache::try_trigger_cleanup() {
+  // See we can take the lock for the notification without blocking.
+  // This allows triggering the cleanup from GC paths, that can hold
+  // the service lock for e.g. oop iteration in service thread.
+  if (has_cleanup_work() && Service_lock->try_lock_without_rank_check()) {
     Service_lock->notify_all();
+    Service_lock->unlock();
   }
 }
 

src/hotspot/share/interpreter/oopMapCache.hpp

@@ -183,8 +183,8 @@ class OopMapCache : public CHeapObj<mtClass> {
   // Check if we need to clean up old entries
   static bool has_cleanup_work();
 
-  // Request cleanup if work is needed
-  static void trigger_cleanup();
+  // Request cleanup if work is needed and notification is currently possible
+  static void try_trigger_cleanup();
 
   // Clean up the old entries
   static void cleanup();