8322692: ZGC: avoid over-unrolling due to hidden barrier size

Reviewed-by: eosterlund, kvn

Author: robcasloz

src/hotspot/share/gc/shared/c2/barrierSetC2.hpp

@@ -278,6 +278,9 @@ class BarrierSetC2: public CHeapObj<mtGC> {
   virtual bool optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { return false; }
   virtual bool strip_mined_loops_expanded(LoopOptsMode mode) const { return false; }
   virtual bool is_gc_specific_loop_opts_pass(LoopOptsMode mode) const { return false; }
+  // Estimated size of the node barrier in number of C2 Ideal nodes.
+  // This is used to guide heuristics in C2, e.g. whether to unroll a loop.
+  virtual uint estimated_barrier_size(const Node* node) const { return 0; }
 
   enum CompilePhase {
     BeforeOptimize,

src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -320,6 +320,20 @@ void ZStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
   ZBarrierSet::assembler()->generate_c2_store_barrier_stub(&masm, static_cast<ZStoreBarrierStubC2*>(this));
 }
 
+uint ZBarrierSetC2::estimated_barrier_size(const Node* node) const {
+  uint8_t barrier_data = MemNode::barrier_data(node);
+  assert(barrier_data != 0, "should be a barrier node");
+  uint uncolor_or_color_size = node->is_Load() ? 1 : 2;
+  if ((barrier_data & ZBarrierElided) != 0) {
+    return uncolor_or_color_size;
+  }
+  // A compare and branch corresponds to approximately four fast-path Ideal
+  // nodes (Cmp, Bool, If, If projection). The slow path (If projection and
+  // runtime call) is excluded since the corresponding code is laid out
+  // separately and does not directly affect performance.
+  return uncolor_or_color_size + 4;
+}
+
 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
   return new (comp_arena) ZBarrierSetC2State(comp_arena);
 }

src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp

@@ -128,6 +128,7 @@ class ZBarrierSetC2 : public BarrierSetC2 {
                                         const Type* val_type) const;
 
 public:
+  virtual uint estimated_barrier_size(const Node* node) const;
   virtual void* create_barrier_state(Arena* comp_arena) const;
   virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc,
                                                BasicType type,

src/hotspot/share/opto/loopTransform.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,8 @@
 
 #include "precompiled.hpp"
 #include "compiler/compileLog.hpp"
+#include "gc/shared/barrierSet.hpp"
+#include "gc/shared/c2/barrierSetC2.hpp"
 #include "memory/allocation.inline.hpp"
 #include "opto/addnode.hpp"
 #include "opto/callnode.hpp"
@@ -996,9 +998,12 @@ bool IdealLoopTree::policy_unroll(PhaseIdealLoop *phase) {
   uint body_size = _body.size();
   // Key test to unroll loop in CRC32 java code
   int xors_in_loop = 0;
-  // Also count ModL, DivL and MulL which expand mightly
+  // Also count ModL, DivL, MulL, and other nodes that expand mightly
   for (uint k = 0; k < _body.size(); k++) {
     Node* n = _body.at(k);
+    if (MemNode::barrier_data(n) != 0) {
+      body_size += BarrierSet::barrier_set()->barrier_set_c2()->estimated_barrier_size(n);
+    }
     switch (n->Opcode()) {
       case Op_XorI: xors_in_loop++; break; // CRC32 java code
       case Op_ModL: body_size += 30; break;

src/hotspot/share/opto/memnode.cpp

@@ -839,6 +839,15 @@ const TypePtr* MemNode::calculate_adr_type(const Type* t, const TypePtr* cross_c
   }
 }
 
+uint8_t MemNode::barrier_data(const Node* n) {
+  if (n->is_LoadStore()) {
+    return n->as_LoadStore()->barrier_data();
+  } else if (n->is_Mem()) {
+    return n->as_Mem()->barrier_data();
+  }
+  return 0;
+}
+
 //=============================================================================
 // Should LoadNode::Ideal() attempt to remove control edges?
 bool LoadNode::can_remove_control() const {

src/hotspot/share/opto/memnode.hpp

@@ -126,6 +126,9 @@ class MemNode : public Node {
 #endif
   }
 
+  // Return the barrier data of n, if available, or 0 otherwise.
+  static uint8_t barrier_data(const Node* n);
+
   // Map a load or store opcode to its corresponding store opcode.
   // (Return -1 if unknown.)
   virtual int store_Opcode() const { return -1; }

test/hotspot/jtreg/compiler/gcbarriers/TestZGCUnrolling.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package compiler.gcbarriers;
+
+import compiler.lib.ir_framework.*;
+import java.lang.invoke.VarHandle;
+import java.lang.invoke.MethodHandles;
+
+/**
+ * @test
+ * @summary Test that the expanded size of ZGC barriers is taken into account in
+ *          C2's loop unrolling heuristics so that over-unrolling is avoided.
+ *          The tests use volatile memory accesses to prevent C2 from simply
+ *          optimizing them away.
+ * @library /test/lib /
+ * @requires vm.gc.ZGenerational
+ * @run driver compiler.gcbarriers.TestZGCUnrolling
+ */
+
+public class TestZGCUnrolling {
+
+    static class Outer {
+        Object f;
+    }
+
+    static final VarHandle fVarHandle;
+    static {
+        MethodHandles.Lookup l = MethodHandles.lookup();
+        try {
+            fVarHandle = l.findVarHandle(Outer.class, "f", Object.class);
+        } catch (Exception e) {
+            throw new Error(e);
+        }
+    }
+
+    public static void main(String[] args) {
+        TestFramework.runWithFlags("-XX:+UseZGC", "-XX:+ZGenerational",
+                                   "-XX:LoopUnrollLimit=24");
+    }
+
+    @Test
+    @IR(counts = {IRNode.STORE_P, "1"})
+    public static void testNoUnrolling(Outer o, Object o1) {
+        for (int i = 0; i < 64; i++) {
+            fVarHandle.setVolatile(o, o1);
+        }
+    }
+
+    @Run(test = {"testNoUnrolling"})
+    void run() {
+        testNoUnrolling(new Outer(), new Object());
+    }
+}