8302850: Implement C1 clone intrinsic that reuses arraycopy code for primitive arrays

Reviewed-by: dlong, roland

Author: galderz

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

@@ -1212,7 +1212,8 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
                       arrayOopDesc::base_offset_in_bytes(op->type()),
                       array_element_size(op->type()),
                       op->klass()->as_register(),
-                      *op->stub()->entry());
+                      *op->stub()->entry(),
+                      op->zero_array());
   }
   __ bind(*op->stub()->continuation());
 }
@@ -2504,7 +2505,9 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
    __ call_VM_leaf(entry, 3);
  }
 
-  __ bind(*stub->continuation());
+  if (stub != nullptr) {
+    __ bind(*stub->continuation());
+  }
 }
 
 

src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp

@@ -878,7 +878,13 @@ void LIRGenerator::do_ArrayCopy(Intrinsic* x) {
   assert(x->number_of_arguments() == 5, "wrong type");
 
   // Make all state_for calls early since they can emit code
-  CodeEmitInfo* info = state_for(x, x->state());
+  CodeEmitInfo* info = nullptr;
+  if (x->state_before() != nullptr && x->state_before()->force_reexecute()) {
+    info = state_for(x, x->state_before());
+    info->set_force_reexecute();
+  } else {
+    info = state_for(x, x->state());
+  }
 
   LIRItem src(x->argument_at(0), this);
   LIRItem src_pos(x->argument_at(1), this);
@@ -911,6 +917,9 @@ void LIRGenerator::do_ArrayCopy(Intrinsic* x) {
   int flags;
   ciArrayKlass* expected_type;
   arraycopy_helper(x, &flags, &expected_type);
+  if (x->check_flag(Instruction::OmitChecksFlag)) {
+    flags = 0;
+  }
 
   __ arraycopy(src.result(), src_pos.result(), dst.result(), dst_pos.result(), length.result(), tmp, expected_type, flags, info); // does add_safepoint
 }
@@ -1132,7 +1141,13 @@ void LIRGenerator::do_NewInstance(NewInstance* x) {
 }
 
 void LIRGenerator::do_NewTypeArray(NewTypeArray* x) {
-  CodeEmitInfo* info = state_for(x, x->state());
+  CodeEmitInfo* info = nullptr;
+  if (x->state_before() != nullptr && x->state_before()->force_reexecute()) {
+    info = state_for(x, x->state_before());
+    info->set_force_reexecute();
+  } else {
+    info = state_for(x, x->state());
+  }
 
   LIRItem length(x->length(), this);
   length.load_item_force(FrameMap::r19_opr);
@@ -1149,7 +1164,7 @@ void LIRGenerator::do_NewTypeArray(NewTypeArray* x) {
   __ metadata2reg(ciTypeArrayKlass::make(elem_type)->constant_encoding(), klass_reg);
 
   CodeStub* slow_path = new NewTypeArrayStub(klass_reg, len, reg, info);
-  __ allocate_array(reg, len, tmp1, tmp2, tmp3, tmp4, elem_type, klass_reg, slow_path);
+  __ allocate_array(reg, len, tmp1, tmp2, tmp3, tmp4, elem_type, klass_reg, slow_path, x->zero_array());
 
   LIR_Opr result = rlock_result(x);
   __ move(reg, result);

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

@@ -272,7 +272,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
 
   verify_oop(obj);
 }
-void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
+void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case, bool zero_array) {
   assert_different_registers(obj, len, t1, t2, klass);
 
   // determine alignment mask
@@ -297,7 +297,9 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
   // following the length field in initialize_header().
   int base_offset = align_up(base_offset_in_bytes, BytesPerWord);
   // clear rest of allocated space
-  initialize_body(obj, arr_size, base_offset, t1, t2);
+  if (zero_array) {
+    initialize_body(obj, arr_size, base_offset, t1, t2);
+  }
   if (Compilation::current()->bailed_out()) {
     return;
   }

src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp

@@ -100,7 +100,8 @@ using MacroAssembler::null_check;
   // base_offset_in_bytes: offset of first array element, in bytes
   // f                   : element scale factor
   // slow_case           : exit to slow case implementation if fast allocation fails
-  void allocate_array(Register obj, Register len, Register t, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case);
+  // zero_array          : zero the allocated array or not
+  void allocate_array(Register obj, Register len, Register t, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case, bool zero_array);
 
   int  rsp_offset() const { return _rsp_offset; }
   void set_rsp_offset(int n) { _rsp_offset = n; }

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

@@ -1621,7 +1621,8 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
                       arrayOopDesc::base_offset_in_bytes(op->type()),
                       array_element_size(op->type()),
                       op->klass()->as_register(),
-                      *op->stub()->entry());
+                      *op->stub()->entry(),
+                      op->zero_array());
   }
   __ bind(*op->stub()->continuation());
 }
@@ -3453,7 +3454,9 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
   __ call_VM_leaf(entry, 0);
 
-  __ bind(*stub->continuation());
+  if (stub != nullptr) {
+    __ bind(*stub->continuation());
+  }
 }
 
 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {

src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp

@@ -1004,7 +1004,13 @@ void LIRGenerator::do_ArrayCopy(Intrinsic* x) {
   assert(x->number_of_arguments() == 5, "wrong type");
 
   // Make all state_for calls early since they can emit code
-  CodeEmitInfo* info = state_for(x, x->state());
+  CodeEmitInfo* info = nullptr;
+  if (x->state_before() != nullptr && x->state_before()->force_reexecute()) {
+    info = state_for(x, x->state_before());
+    info->set_force_reexecute();
+  } else {
+    info = state_for(x, x->state());
+  }
 
   LIRItem src(x->argument_at(0), this);
   LIRItem src_pos(x->argument_at(1), this);
@@ -1016,13 +1022,25 @@ void LIRGenerator::do_ArrayCopy(Intrinsic* x) {
   // LinearScan will fail allocation (because arraycopy always needs a
   // call)
 
+  int flags;
+  ciArrayKlass* expected_type;
+  arraycopy_helper(x, &flags, &expected_type);
+  if (x->check_flag(Instruction::OmitChecksFlag)) {
+    flags = 0;
+  }
+
 #ifndef _LP64
   src.load_item_force     (FrameMap::rcx_oop_opr);
   src_pos.load_item_force (FrameMap::rdx_opr);
   dst.load_item_force     (FrameMap::rax_oop_opr);
   dst_pos.load_item_force (FrameMap::rbx_opr);
   length.load_item_force  (FrameMap::rdi_opr);
   LIR_Opr tmp =           (FrameMap::rsi_opr);
+
+  if (expected_type != nullptr && flags == 0) {
+    FrameMap* f = Compilation::current()->frame_map();
+    f->update_reserved_argument_area_size(3 * BytesPerWord);
+  }
 #else
 
   // The java calling convention will give us enough registers
@@ -1044,10 +1062,6 @@ void LIRGenerator::do_ArrayCopy(Intrinsic* x) {
 
   set_no_result(x);
 
-  int flags;
-  ciArrayKlass* expected_type;
-  arraycopy_helper(x, &flags, &expected_type);
-
   __ arraycopy(src.result(), src_pos.result(), dst.result(), dst_pos.result(), length.result(), tmp, expected_type, flags, info); // does add_safepoint
 }
 
@@ -1310,7 +1324,13 @@ void LIRGenerator::do_NewInstance(NewInstance* x) {
 
 
 void LIRGenerator::do_NewTypeArray(NewTypeArray* x) {
-  CodeEmitInfo* info = state_for(x, x->state());
+  CodeEmitInfo* info = nullptr;
+  if (x->state_before() != nullptr && x->state_before()->force_reexecute()) {
+    info = state_for(x, x->state_before());
+    info->set_force_reexecute();
+  } else {
+    info = state_for(x, x->state());
+  }
 
   LIRItem length(x->length(), this);
   length.load_item_force(FrameMap::rbx_opr);
@@ -1327,7 +1347,7 @@ void LIRGenerator::do_NewTypeArray(NewTypeArray* x) {
   __ metadata2reg(ciTypeArrayKlass::make(elem_type)->constant_encoding(), klass_reg);
 
   CodeStub* slow_path = new NewTypeArrayStub(klass_reg, len, reg, info);
-  __ allocate_array(reg, len, tmp1, tmp2, tmp3, tmp4, elem_type, klass_reg, slow_path);
+  __ allocate_array(reg, len, tmp1, tmp2, tmp3, tmp4, elem_type, klass_reg, slow_path, x->zero_array());
 
   LIR_Opr result = rlock_result(x);
   __ move(reg, result);

src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp

@@ -278,7 +278,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
   verify_oop(obj);
 }
 
-void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, Address::ScaleFactor f, Register klass, Label& slow_case) {
+void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, Address::ScaleFactor f, Register klass, Label& slow_case, bool zero_array) {
   assert(obj == rax, "obj must be in rax, for cmpxchg");
   assert_different_registers(obj, len, t1, t2, klass);
 
@@ -300,11 +300,13 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
   initialize_header(obj, klass, len, t1, t2);
 
   // clear rest of allocated space
-  const Register len_zero = len;
-  // Align-up to word boundary, because we clear the 4 bytes potentially
-  // following the length field in initialize_header().
-  int base_offset = align_up(base_offset_in_bytes, BytesPerWord);
-  initialize_body(obj, arr_size, base_offset, len_zero);
+  if (zero_array) {
+    const Register len_zero = len;
+    // Align-up to word boundary, because we clear the 4 bytes potentially
+    // following the length field in initialize_header().
+    int base_offset = align_up(base_offset_in_bytes, BytesPerWord);
+    initialize_body(obj, arr_size, base_offset, len_zero);
+  }
 
   if (CURRENT_ENV->dtrace_alloc_probes()) {
     assert(obj == rax, "must be");

src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp

@@ -89,7 +89,8 @@
   // base_offset_in_bytes: offset of the first array element, in bytes
   // f                   : element scale factor
   // slow_case           : exit to slow case implementation if fast allocation fails
-  void allocate_array(Register obj, Register len, Register t, Register t2, int base_offset_in_bytes, Address::ScaleFactor f, Register klass, Label& slow_case);
+  // zero_array          : zero the allocated array or not
+  void allocate_array(Register obj, Register len, Register t, Register t2, int base_offset_in_bytes, Address::ScaleFactor f, Register klass, Label& slow_case, bool zero_array);
 
   int  rsp_offset() const { return _rsp_offset; }
   void set_rsp_offset(int n) { _rsp_offset = n; }

src/hotspot/share/c1/c1_Compiler.cpp

@@ -235,6 +235,9 @@ bool Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
   case vmIntrinsics::_counterTime:
 #endif
   case vmIntrinsics::_getObjectSize:
+#if defined(X86) || defined(AARCH64)
+  case vmIntrinsics::_clone:
+#endif
     break;
   case vmIntrinsics::_blackhole:
     break;

src/hotspot/share/c1/c1_GraphBuilder.cpp

@@ -2026,8 +2026,11 @@ void GraphBuilder::invoke(Bytecodes::Code code) {
       int index = state()->stack_size() - (target->arg_size_no_receiver() + 1);
       receiver = state()->stack_at(index);
       ciType* type = receiver->exact_type();
-      if (type != nullptr && type->is_loaded() &&
-          type->is_instance_klass() && !type->as_instance_klass()->is_interface()) {
+      if (type != nullptr && type->is_loaded()) {
+        assert(!type->is_instance_klass() || !type->as_instance_klass()->is_interface(), "Must not be an interface");
+        // Detects non-interface instances, primitive arrays, and some object arrays.
+        // Array receivers can only call Object methods, so we should be able to allow
+        // all object arrays here too, even those with unloaded types.
         receiver_klass = (ciInstanceKlass*) type;
         type_is_exact = true;
       }
@@ -2243,7 +2246,7 @@ void GraphBuilder::new_instance(int klass_index) {
 
 void GraphBuilder::new_type_array() {
   ValueStack* state_before = copy_state_exhandling();
-  apush(append_split(new NewTypeArray(ipop(), (BasicType)stream()->get_index(), state_before)));
+  apush(append_split(new NewTypeArray(ipop(), (BasicType)stream()->get_index(), state_before, true)));
 }
 
 
@@ -3650,9 +3653,13 @@ void GraphBuilder::build_graph_for_intrinsic(ciMethod* callee, bool ignore_retur
   case vmIntrinsics::_getAndSetReference     : append_unsafe_get_and_set(callee, false); return;
   case vmIntrinsics::_getCharStringU         : append_char_access(callee, false); return;
   case vmIntrinsics::_putCharStringU         : append_char_access(callee, true); return;
+  case vmIntrinsics::_clone                  : append_alloc_array_copy(callee); return;
   default:
     break;
   }
+  if (_inline_bailout_msg != nullptr) {
+    return;
+  }
 
   // create intrinsic node
   const bool has_receiver = !callee->is_static();
@@ -3714,6 +3721,9 @@ bool GraphBuilder::try_inline_intrinsics(ciMethod* callee, bool ignore_return) {
     }
   }
   build_graph_for_intrinsic(callee, ignore_return);
+  if (_inline_bailout_msg != nullptr) {
+    return false;
+  }
   return true;
 }
 
@@ -4427,6 +4437,43 @@ void GraphBuilder::append_char_access(ciMethod* callee, bool is_store) {
   }
 }
 
+void GraphBuilder::append_alloc_array_copy(ciMethod* callee) {
+  const int args_base = state()->stack_size() - callee->arg_size();
+  ciType* receiver_type = state()->stack_at(args_base)->exact_type();
+  if (receiver_type == nullptr) {
+    inline_bailout("must have a receiver");
+    return;
+  }
+  if (!receiver_type->is_type_array_klass()) {
+    inline_bailout("clone array not primitive");
+    return;
+  }
+
+  ValueStack* state_before = copy_state_before();
+  state_before->set_force_reexecute();
+  Value src = apop();
+  BasicType basic_type = src->exact_type()->as_array_klass()->element_type()->basic_type();
+  Value length = append(new ArrayLength(src, state_before));
+  Value new_array = append_split(new NewTypeArray(length, basic_type, state_before, false));
+
+  ValueType* result_type = as_ValueType(callee->return_type());
+  vmIntrinsics::ID id = vmIntrinsics::_arraycopy;
+  Values* args = new Values(5);
+  args->push(src);
+  args->push(append(new Constant(new IntConstant(0))));
+  args->push(new_array);
+  args->push(append(new Constant(new IntConstant(0))));
+  args->push(length);
+  const bool has_receiver = true;
+  Intrinsic* array_copy = new Intrinsic(result_type, id,
+                                    args, has_receiver, state_before,
+                                    vmIntrinsics::preserves_state(id),
+                                    vmIntrinsics::can_trap(id));
+  array_copy->set_flag(Instruction::OmitChecksFlag, true);
+  append_split(array_copy);
+  apush(new_array);
+}
+
 void GraphBuilder::print_inlining(ciMethod* callee, const char* msg, bool success) {
   CompileLog* log = compilation()->log();
   if (log != nullptr) {

src/hotspot/share/c1/c1_GraphBuilder.hpp

@@ -379,6 +379,7 @@ class GraphBuilder {
   void append_unsafe_CAS(ciMethod* callee);
   void append_unsafe_get_and_set(ciMethod* callee, bool is_add);
   void append_char_access(ciMethod* callee, bool is_store);
+  void append_alloc_array_copy(ciMethod* callee);
 
   void print_inlining(ciMethod* callee, const char* msg, bool success = true);
 

src/hotspot/share/c1/c1_Instruction.hpp

@@ -364,6 +364,7 @@ class Instruction: public CompilationResourceObj {
     InWorkListFlag,
     DeoptimizeOnException,
     KillsMemoryFlag,
+    OmitChecksFlag,
     InstructionLastFlag
   };
 
@@ -1327,16 +1328,19 @@ BASE(NewArray, StateSplit)
 LEAF(NewTypeArray, NewArray)
  private:
   BasicType _elt_type;
+  bool _zero_array;
 
  public:
   // creation
-  NewTypeArray(Value length, BasicType elt_type, ValueStack* state_before)
+  NewTypeArray(Value length, BasicType elt_type, ValueStack* state_before, bool zero_array)
   : NewArray(length, state_before)
   , _elt_type(elt_type)
+  , _zero_array(zero_array)
   {}
 
   // accessors
   BasicType elt_type() const                     { return _elt_type; }
+  bool zero_array()    const                     { return _zero_array; }
   ciType* exact_type() const;
 };