Merge jdk19

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -723,7 +723,7 @@ void MacroAssembler::emit_static_call_stub() {
   isb();
   mov_metadata(rmethod, (Metadata*)NULL);
 
-  // Jump to the entry point of the i2c stub.
+  // Jump to the entry point of the c2i stub.
   movptr(rscratch1, 0);
   br(rscratch1);
 }

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

@@ -1013,30 +1013,72 @@ static void gen_continuation_enter(MacroAssembler* masm,
                                  int& exception_offset,
                                  OopMapSet*oop_maps,
                                  int& frame_complete,
-                                 int& stack_slots) {
+                                 int& stack_slots,
+                                 int& interpreted_entry_offset,
+                                 int& compiled_entry_offset) {
   //verify_oop_args(masm, method, sig_bt, regs);
-  Address resolve(SharedRuntime::get_resolve_static_call_stub(),
-                  relocInfo::static_call_type);
+  Address resolve(SharedRuntime::get_resolve_static_call_stub(), relocInfo::static_call_type);
 
-  stack_slots = 2; // will be overwritten
   address start = __ pc();
 
   Label call_thaw, exit;
 
-  __ enter();
+  // i2i entry used at interp_only_mode only
+  interpreted_entry_offset = __ pc() - start;
+  {
 
-  OopMap* map = continuation_enter_setup(masm, stack_slots);
+#ifdef ASSERT
+    Label is_interp_only;
+    __ ldrw(rscratch1, Address(rthread, JavaThread::interp_only_mode_offset()));
+    __ cbnzw(rscratch1, is_interp_only);
+    __ stop("enterSpecial interpreter entry called when not in interp_only_mode");
+    __ bind(is_interp_only);
+#endif
 
-  // Frame is now completed as far as size and linkage.
-  frame_complete =__ pc() - start;
+    // Read interpreter arguments into registers (this is an ad-hoc i2c adapter)
+    __ ldr(c_rarg1, Address(esp, Interpreter::stackElementSize*2));
+    __ ldr(c_rarg2, Address(esp, Interpreter::stackElementSize*1));
+    __ ldr(c_rarg3, Address(esp, Interpreter::stackElementSize*0));
+    __ push_cont_fastpath(rthread);
+
+    __ enter();
+    stack_slots = 2; // will be adjusted in setup
+    OopMap* map = continuation_enter_setup(masm, stack_slots);
+    // The frame is complete here, but we only record it for the compiled entry, so the frame would appear unsafe,
+    // but that's okay because at the very worst we'll miss an async sample, but we're in interp_only_mode anyway.
+
+    fill_continuation_entry(masm);
+
+    __ cmp(c_rarg2, (u1)0);
+    __ br(Assembler::NE, call_thaw);
+
+    address mark = __ pc();
+    __ trampoline_call1(resolve, NULL, false);
+
+    oop_maps->add_gc_map(__ pc() - start, map);
+    __ post_call_nop();
+
+    __ b(exit);
+
+    CodeBuffer* cbuf = masm->code_section()->outer();
+    CompiledStaticCall::emit_to_interp_stub(*cbuf, mark);
+  }
+
+  // compiled entry
+  __ align(CodeEntryAlignment);
+  compiled_entry_offset = __ pc() - start;
+
+  __ enter();
+  stack_slots = 2; // will be adjusted in setup
+  OopMap* map = continuation_enter_setup(masm, stack_slots);
+  frame_complete = __ pc() - start;
 
   fill_continuation_entry(masm);
 
   __ cmp(c_rarg2, (u1)0);
   __ br(Assembler::NE, call_thaw);
 
   address mark = __ pc();
-
   __ trampoline_call1(resolve, NULL, false);
 
   oop_maps->add_gc_map(__ pc() - start, map);
@@ -1079,7 +1121,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
   }
 
   CodeBuffer* cbuf = masm->code_section()->outer();
-  address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, mark);
+  CompiledStaticCall::emit_to_interp_stub(*cbuf, mark);
 }
 
 static void gen_special_dispatch(MacroAssembler* masm,
@@ -1169,19 +1211,22 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
   if (method->is_continuation_enter_intrinsic()) {
     vmIntrinsics::ID iid = method->intrinsic_id();
     intptr_t start = (intptr_t)__ pc();
-    int vep_offset = ((intptr_t)__ pc()) - start;
+    int vep_offset = 0;
     int exception_offset = 0;
     int frame_complete = 0;
     int stack_slots = 0;
     OopMapSet* oop_maps =  new OopMapSet();
+    int interpreted_entry_offset = -1;
     gen_continuation_enter(masm,
                          method,
                          in_sig_bt,
                          in_regs,
                          exception_offset,
                          oop_maps,
                          frame_complete,
-                         stack_slots);
+                         stack_slots,
+                         interpreted_entry_offset,
+                         vep_offset);
     __ flush();
     nmethod* nm = nmethod::new_native_nmethod(method,
                                               compile_id,
@@ -1193,7 +1238,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
                                               in_ByteSize(-1),
                                               oop_maps,
                                               exception_offset);
-    ContinuationEntry::set_enter_code(nm);
+    ContinuationEntry::set_enter_code(nm, interpreted_entry_offset);
     return nm;
   }
 

src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp

@@ -1276,7 +1276,9 @@ static void gen_continuation_enter(MacroAssembler* masm,
                                  int& exception_offset,
                                  OopMapSet* oop_maps,
                                  int& frame_complete,
-                                 int& stack_slots) {
+                                 int& stack_slots,
+                                 int& interpreted_entry_offset,
+                                 int& compiled_entry_offset) {
 
   // enterSpecial(Continuation c, boolean isContinue, boolean isVirtualThread)
   int pos_cont_obj   = 0;
@@ -1298,8 +1300,68 @@ static void gen_continuation_enter(MacroAssembler* masm,
   // Utility methods kill rax, make sure there are no collisions
   assert_different_registers(rax, reg_cont_obj, reg_is_cont, reg_is_virtual);
 
+  AddressLiteral resolve(SharedRuntime::get_resolve_static_call_stub(),
+                         relocInfo::static_call_type);
+
   address start = __ pc();
 
+  Label L_thaw, L_exit;
+
+  // i2i entry used at interp_only_mode only
+  interpreted_entry_offset = __ pc() - start;
+  {
+#ifdef ASSERT
+    Label is_interp_only;
+    __ cmpb(Address(r15_thread, JavaThread::interp_only_mode_offset()), 0);
+    __ jcc(Assembler::notEqual, is_interp_only);
+    __ stop("enterSpecial interpreter entry called when not in interp_only_mode");
+    __ bind(is_interp_only);
+#endif
+
+    __ pop(rax); // return address
+    // Read interpreter arguments into registers (this is an ad-hoc i2c adapter)
+    __ movptr(c_rarg1, Address(rsp, Interpreter::stackElementSize*2));
+    __ movl(c_rarg2,   Address(rsp, Interpreter::stackElementSize*1));
+    __ movl(c_rarg3,   Address(rsp, Interpreter::stackElementSize*0));
+    __ andptr(rsp, -16); // Ensure compiled code always sees stack at proper alignment
+    __ push(rax); // return address
+    __ push_cont_fastpath();
+
+    __ enter();
+
+    stack_slots = 2; // will be adjusted in setup
+    OopMap* map = continuation_enter_setup(masm, stack_slots);
+    // The frame is complete here, but we only record it for the compiled entry, so the frame would appear unsafe,
+    // but that's okay because at the very worst we'll miss an async sample, but we're in interp_only_mode anyway.
+
+    __ verify_oop(reg_cont_obj);
+
+    fill_continuation_entry(masm, reg_cont_obj, reg_is_virtual);
+
+    // If continuation, call to thaw. Otherwise, resolve the call and exit.
+    __ testptr(reg_is_cont, reg_is_cont);
+    __ jcc(Assembler::notZero, L_thaw);
+
+    // --- Resolve path
+
+    // Make sure the call is patchable
+    __ align(BytesPerWord, __ offset() + NativeCall::displacement_offset);
+    // Emit stub for static call
+    CodeBuffer* cbuf = masm->code_section()->outer();
+    address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, __ pc());
+    if (stub == nullptr) {
+      fatal("CodeCache is full at gen_continuation_enter");
+    }
+    __ call(resolve);
+    oop_maps->add_gc_map(__ pc() - start, map);
+    __ post_call_nop();
+
+    __ jmp(L_exit);
+  }
+
+  // compiled entry
+  __ align(CodeEntryAlignment);
+  compiled_entry_offset = __ pc() - start;
   __ enter();
 
   stack_slots = 2; // will be adjusted in setup
@@ -1312,8 +1374,6 @@ static void gen_continuation_enter(MacroAssembler* masm,
 
   fill_continuation_entry(masm, reg_cont_obj, reg_is_virtual);
 
-  Label L_thaw, L_exit;
-
   // If isContinue, call to thaw. Otherwise, call Continuation.enter(Continuation c, boolean isContinue)
   __ testptr(reg_is_cont, reg_is_cont);
   __ jccb(Assembler::notZero, L_thaw);
@@ -1334,8 +1394,6 @@ static void gen_continuation_enter(MacroAssembler* masm,
   // SharedRuntime::find_callee_info_helper() which calls
   // LinkResolver::resolve_continuation_enter() which resolves the call to
   // Continuation.enter(Continuation c, boolean isContinue).
-  AddressLiteral resolve(SharedRuntime::get_resolve_static_call_stub(),
-                         relocInfo::static_call_type);
   __ call(resolve);
 
   oop_maps->add_gc_map(__ pc() - start, map);
@@ -1474,17 +1532,20 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
   if (method->is_continuation_enter_intrinsic()) {
     vmIntrinsics::ID iid = method->intrinsic_id();
     intptr_t start = (intptr_t)__ pc();
-    int vep_offset = ((intptr_t)__ pc()) - start;
+    int vep_offset = 0;
     int exception_offset = 0;
     int frame_complete = 0;
     int stack_slots = 0;
     OopMapSet* oop_maps =  new OopMapSet();
+    int interpreted_entry_offset = -1;
     gen_continuation_enter(masm,
                          in_regs,
                          exception_offset,
                          oop_maps,
                          frame_complete,
-                         stack_slots);
+                         stack_slots,
+                         interpreted_entry_offset,
+                         vep_offset);
     __ flush();
     nmethod* nm = nmethod::new_native_nmethod(method,
                                               compile_id,
@@ -1496,7 +1557,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
                                               in_ByteSize(-1),
                                               oop_maps,
                                               exception_offset);
-    ContinuationEntry::set_enter_code(nm);
+    ContinuationEntry::set_enter_code(nm, interpreted_entry_offset);
     return nm;
   }
 

src/hotspot/share/code/compiledIC.cpp

@@ -39,6 +39,7 @@
 #include "oops/method.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "oops/symbol.hpp"
+#include "runtime/continuationEntry.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/icache.hpp"
 #include "runtime/safepoint.hpp"
@@ -653,6 +654,13 @@ void CompiledStaticCall::compute_entry(const methodHandle& m, bool caller_is_nme
   }
 }
 
+void CompiledStaticCall::compute_entry_for_continuation_entry(const methodHandle& m, StaticCallInfo& info) {
+  if (ContinuationEntry::is_interpreted_call(instruction_address())) {
+    info._to_interpreter = true;
+    info._entry = m()->get_c2i_entry();
+  }
+}
+
 address CompiledDirectStaticCall::find_stub_for(address instruction) {
   // Find reloc. information containing this call-site
   RelocIterator iter((nmethod*)NULL, instruction);

src/hotspot/share/code/compiledIC.hpp

@@ -345,6 +345,7 @@ class CompiledStaticCall : public ResourceObj {
 
   // Compute entry point given a method
   static void compute_entry(const methodHandle& m, bool caller_is_nmethod, StaticCallInfo& info);
+  void compute_entry_for_continuation_entry(const methodHandle& m, StaticCallInfo& info);
 
 public:
   // Clean static call (will force resolving on next use)

src/hotspot/share/oops/method.cpp

@@ -61,6 +61,7 @@
 #include "prims/methodHandles.hpp"
 #include "runtime/arguments.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/continuationEntry.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/init.hpp"
@@ -1328,7 +1329,7 @@ void Method::set_code(const methodHandle& mh, CompiledMethod *code) {
     assert(mh->_from_interpreted_entry == NULL, "initialized incorrectly"); // see link_method
 
     // This is the entry used when we're in interpreter-only mode; see InterpreterMacroAssembler::jump_from_interpreted
-    mh->_i2i_entry = mh->get_i2c_entry();
+    mh->_i2i_entry = ContinuationEntry::interpreted_entry();
     // This must come last, as it is what's tested in LinkResolver::resolve_static_call
     Atomic::release_store(&mh->_from_interpreted_entry , mh->get_i2c_entry());
   } else if (!mh->is_method_handle_intrinsic()) {

src/hotspot/share/runtime/continuationEntry.cpp

@@ -23,6 +23,7 @@
  */
 
 #include "precompiled.hpp"
+#include "code/compiledIC.hpp"
 #include "code/nmethod.hpp"
 #include "runtime/continuation.hpp"
 #include "runtime/continuationEntry.inline.hpp"
@@ -34,10 +35,32 @@
 
 int ContinuationEntry::_return_pc_offset = 0;
 address ContinuationEntry::_return_pc = nullptr;
+CompiledMethod* ContinuationEntry::_enter_special = nullptr;
+int ContinuationEntry::_interpreted_entry_offset = 0;
 
-void ContinuationEntry::set_enter_code(CompiledMethod* cm) {
+void ContinuationEntry::set_enter_code(CompiledMethod* cm, int interpreted_entry_offset) {
   assert(_return_pc_offset != 0, "");
   _return_pc = cm->code_begin() + _return_pc_offset;
+
+  _enter_special = cm;
+  _interpreted_entry_offset = interpreted_entry_offset;
+  assert(_enter_special->code_contains(compiled_entry()),    "entry not in enterSpecial");
+  assert(_enter_special->code_contains(interpreted_entry()), "entry not in enterSpecial");
+  assert(interpreted_entry() < compiled_entry(), "unexpected code layout");
+}
+
+address ContinuationEntry::compiled_entry() {
+  return _enter_special->verified_entry_point();
+}
+
+address ContinuationEntry::interpreted_entry() {
+  return _enter_special->code_begin() + _interpreted_entry_offset;
+}
+
+bool ContinuationEntry::is_interpreted_call(address call_address) {
+  assert(_enter_special->code_contains(call_address), "call not in enterSpecial");
+  assert(call_address >= interpreted_entry(), "unexpected location");
+  return call_address < compiled_entry();
 }
 
 ContinuationEntry* ContinuationEntry::from_frame(const frame& f) {

src/hotspot/share/runtime/continuationEntry.hpp

@@ -53,10 +53,13 @@ class ContinuationEntry {
 
 public:
   static int _return_pc_offset; // friend gen_continuation_enter
-  static void set_enter_code(CompiledMethod* nm); // friend SharedRuntime::generate_native_wrapper
+  static void set_enter_code(CompiledMethod* cm, int interpreted_entry_offset);
+  static bool is_interpreted_call(address call_address);
 
 private:
   static address _return_pc;
+  static CompiledMethod* _enter_special;
+  static int _interpreted_entry_offset;
 
 private:
   ContinuationEntry* _parent;
@@ -90,6 +93,11 @@ class ContinuationEntry {
   intptr_t* entry_sp() const { return (intptr_t*)this; }
   intptr_t* entry_fp() const;
 
+  static address compiled_entry();
+  static address interpreted_entry();
+
+  static CompiledMethod* enter_special() { return _enter_special; }
+
   int argsize() const { return _argsize; }
   void set_argsize(int value) { _argsize = value; }