8304310: Initial compilers and runtime handling for multifield backed vectors.

src/hotspot/cpu/x86/interp_masm_x86.cpp

@@ -1179,17 +1179,18 @@ void InterpreterMacroAssembler::remove_activation(
     Label skip;
     test_oop_is_not_inline_type(rax, rscratch1, skip);
 
-#ifndef _LP64
-    super_call_VM_leaf(StubRoutines::load_inline_type_fields_in_regs());
-#else
-    // Load fields from a buffered value with an inline class specific handler
+    // Skip scalarization for vector value objects (concrete vectors and payloads).
     load_klass(rdi, rax, rscratch1);
     movptr(rscratch1, rax);
-    // Skip scalarization for vector value objects (concrete vectors and payloads).
-    super_call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::is_vector_value_instance), rdi);
+    super_call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::skip_value_scalarization), rdi);
     testptr(rax, rax);
     movptr(rax, rscratch1);
     jcc(Assembler::notZero, skip);
+
+#ifndef _LP64
+    super_call_VM_leaf(StubRoutines::load_inline_type_fields_in_regs());
+#else
+    // Load fields from a buffered value with an inline class specific handler
     load_klass(rdi, rax, rscratch1);
     movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
     movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));

src/hotspot/share/ci/ciInlineKlass.cpp

@@ -92,12 +92,12 @@ bool ciInlineKlass::flatten_array() const {
 
 // Can this inline type be passed as multiple values?
 bool ciInlineKlass::can_be_passed_as_fields() const {
-  GUARDED_VM_ENTRY(return !VectorSupport::skip_value_scalarization(const_cast<ciInlineKlass*>(this)) && to_InlineKlass()->can_be_passed_as_fields();)
+  GUARDED_VM_ENTRY(return to_InlineKlass()->can_be_passed_as_fields();)
 }
 
 // Can this inline type be returned as multiple values?
 bool ciInlineKlass::can_be_returned_as_fields() const {
-  GUARDED_VM_ENTRY(return !VectorSupport::skip_value_scalarization(const_cast<ciInlineKlass*>(this)) && to_InlineKlass()->can_be_returned_as_fields();)
+  GUARDED_VM_ENTRY(return to_InlineKlass()->can_be_returned_as_fields();)
 }
 
 bool ciInlineKlass::is_empty() {

src/hotspot/share/opto/compile.cpp

@@ -2756,10 +2756,6 @@ void Compile::Optimize() {
     set_for_igvn(old_worklist); // new_worklist is dead beyond this point
   }
 
-  // Now that all inlining is over and no PhaseRemoveUseless will run, cut edge from root to loop
-  // safepoints
-  remove_root_to_sfpts_edges(igvn);
-
   // Process inline type nodes now that all inlining is over
   process_inline_types(igvn);
 
@@ -2773,6 +2769,9 @@ void Compile::Optimize() {
   }
   assert(!has_vbox_nodes(), "sanity");
 
+  // Now that all inlining is over and no PhaseRemoveUseless will run, cut edge from root to loop
+  // safepoints
+  remove_root_to_sfpts_edges(igvn);
 
   adjust_flattened_array_access_aliases(igvn);
 

src/hotspot/share/opto/inlinetypenode.cpp

@@ -504,7 +504,6 @@ void InlineTypeNode::store(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass
       int vec_len = ft->bundle_size();
       BasicType bt = type2field[ft->basic_type()];
       const Type* val_type = Type::get_const_type(ft);
-      const TypeAryPtr* ary_type = kit->gvn().type(base)->isa_aryptr();
       const TypePtr* adr_type = field_adr_type(base, offset, holder, decorators, kit->gvn());
       Node* adr = kit->basic_plus_adr(base, ptr, offset);
       assert(is_java_primitive(bt) || adr->bottom_type()->is_ptr_to_narrowoop() == UseCompressedOops, "inconsistent");
@@ -513,6 +512,7 @@ void InlineTypeNode::store(GraphKit* kit, Node* base, Node* ptr, ciInstanceKlass
         Node* store = kit->gvn().transform(StoreVectorNode::make(0, kit->control(), kit->memory(adr), adr, adr_type, value, vec_len));
         kit->set_memory(store, adr_type);
       } else {
+        const TypeAryPtr* ary_type = kit->gvn().type(base)->isa_aryptr();
         if (ary_type != NULL) {
           decorators |= IS_ARRAY;
         }

src/hotspot/share/opto/library_call.cpp

@@ -2331,7 +2331,7 @@ bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, c
         // Skip over direct field access for VectorPayloadMF* class instancs since
         // multifield is loaded into vector, alternatively we can create a lane
         // extraction logic.
-        if (field != NULL && !VectorSupport::is_vector_payload_mf(vk)) {
+        if (field != NULL && !VectorSupport::is_vector_payload_mf(vk->get_InlineKlass())) {
           BasicType bt = field->layout_type();
           if (bt == T_ARRAY || bt == T_NARROWOOP || (bt == T_PRIMITIVE_OBJECT && !field->is_flattened())) {
             bt = T_OBJECT;

src/hotspot/share/opto/parse1.cpp

@@ -921,7 +921,7 @@ void Compile::return_values(JVMState* jvms) {
     kit.inc_sp(-ret_size);  // pop the return value(s)
     kit.sync_jvms();
     Node* res = kit.argument(0);
-    if (res->isa_InlineType() && VectorSupport::skip_value_scalarization(res->as_InlineType()->inline_klass())) {
+    if (res->isa_InlineType() && VectorSupport::skip_value_scalarization(res->as_InlineType()->inline_klass()->get_InlineKlass())) {
       InlineTypeNode* vt = res->as_InlineType();
       assert(vt->is_buffered(), "");
       ret->add_req(vt->get_oop());
@@ -2357,7 +2357,7 @@ void Parse::return_current(Node* value) {
       if (!value->is_InlineType()) {
         value = InlineTypeNode::make_from_oop(this, value, return_type->inline_klass(), method()->signature()->returns_null_free_inline_type());
       }
-      if (VectorSupport::skip_value_scalarization(value->as_InlineType()->inline_klass())) {
+      if (VectorSupport::skip_value_scalarization(value->as_InlineType()->inline_klass()->get_InlineKlass())) {
         // Buffer the vector return types, for regular inline object caller expects
         // scalarized fields to be passed back.
         PreserveReexecuteState preexecs(this);

src/hotspot/share/opto/vector.cpp

@@ -34,15 +34,19 @@
 #include "prims/vectorSupport.hpp"
 
 static bool is_vector(ciKlass* klass) {
-  return VectorSupport::is_vector(klass);
+  return klass->is_subclass_of(ciEnv::current()->vector_Vector_klass());
+}
+
+static bool is_vector_payload_mf(ciKlass* klass) {
+  return klass->is_subclass_of(ciEnv::current()->vector_VectorPayloadMF_klass());
 }
 
 static bool is_vector_mask(ciKlass* klass) {
-  return VectorSupport::is_vector_mask(klass);
+  return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
 }
 
 static bool is_vector_shuffle(ciKlass* klass) {
-  return VectorSupport::is_vector_shuffle(klass);
+  return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
 }
 
 void PhaseVector::optimize_vector_boxes() {
@@ -316,23 +320,22 @@ void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) {
   if (vec_box->outcnt() > 0) {
     Node* vbox = vec_box->get_oop();
     Node* vect = vec_box->get_vec();
-    Node* result = expand_vbox_node_helper(vec_box, vbox, vect, vec_box->box_type(), vec_box->vec_type());
+    Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(), vec_box->vec_type());
     C->gvn_replace_by(vec_box, result);
     C->print_method(PHASE_EXPAND_VBOX, 3, vec_box);
   }
   C->remove_macro_node(vec_box);
 }
 
-Node* PhaseVector::expand_vbox_node_helper(Node* vec_box,
-                                           Node* vbox,
+Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
                                            Node* vect,
                                            const TypeInstPtr* box_type,
                                            const TypeVect* vect_type) {
   if (vbox->is_Phi() && vect->is_Phi()) {
     assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), "");
     Node* new_phi = new PhiNode(vbox->as_Phi()->region(), box_type);
     for (uint i = 1; i < vbox->req(); i++) {
-      Node* new_box = expand_vbox_node_helper(vec_box, vbox->in(i), vect->in(i), box_type, vect_type);
+      Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i), box_type, vect_type);
       new_phi->set_req(i, new_box);
     }
     new_phi = C->initial_gvn()->transform(new_phi);
@@ -347,7 +350,7 @@ Node* PhaseVector::expand_vbox_node_helper(Node* vec_box,
     // move up and are guaranteed to dominate.
     Node* new_phi = new PhiNode(vbox->as_Phi()->region(), box_type);
     for (uint i = 1; i < vbox->req(); i++) {
-      Node* new_box = expand_vbox_node_helper(vec_box, vbox->in(i), vect, box_type, vect_type);
+      Node* new_box = expand_vbox_node_helper(vbox->in(i), vect, box_type, vect_type);
       new_phi->set_req(i, new_box);
     }
     new_phi = C->initial_gvn()->transform(new_phi);
@@ -380,12 +383,12 @@ Node* PhaseVector::expand_vbox_alloc_node_vector(VectorBoxAllocateNode* vbox_all
   Node* klass_node = kit.makecon(klass_type);
   Node* buffer_mem = kit.new_instance(klass_node, NULL, NULL, /* deoptimize_on_exception */ true);
 
-  assert(VectorSupport::is_vector(box_klass), "");
+  assert(is_vector(box_klass), "");
   ciField* payload_field = box_klass->declared_nonstatic_field_at(0);
   int offset = payload_field->offset();
   if (!payload_field->is_flattened()) {
     ciInlineKlass* payload_klass = static_cast<ciInlineKlass*>(payload_field->type());
-    assert(VectorSupport::is_vector_payload_mf(payload_klass), "");
+    assert(is_vector_payload_mf(payload_klass), "");
     ciField* mutifield = payload_klass->declared_nonstatic_field_at(0);
     offset += mutifield->offset();
   }
@@ -515,41 +518,7 @@ void PhaseVector::expand_vunbox_node_vector(VectorUnboxNode* vec_unbox) {
     PhaseGVN& gvn = kit.gvn();
 
     Node* vec_val_load = get_loaded_payload(vec_unbox);
-
-    if (vec_val_load == NULL) {
-      Node* obj = vec_unbox->obj();
-      const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr();
-      assert(VectorSupport::is_vector(tinst->instance_klass()), "");
-      ciInlineKlass* box_klass = static_cast<ciInlineKlass*>(tinst->instance_klass());
-
-      const TypeVect* vt = vec_unbox->bottom_type()->is_vect();
-      BasicType bt = vt->element_basic_type();
-      int num_elem = vt->length();
-
-      assert(VectorSupport::is_vector(box_klass), "");
-      ciField* payload_field = box_klass->declared_nonstatic_field_at(0);
-      int offset = payload_field->offset();
-      if (!payload_field->is_flattened()) {
-        ciInlineKlass* payload_klass = static_cast<ciInlineKlass*>(payload_field->type());
-        assert(VectorSupport::is_vector_payload_mf(payload_klass), "");
-        ciField* mutifield = payload_klass->declared_nonstatic_field_at(0);
-        offset += mutifield->offset();
-      }
-
-      Node* mem = vec_unbox->mem();
-      Node* ctrl = vec_unbox->in(0);
-      Node* vec_adr = gvn.transform(kit.basic_plus_adr(obj, offset));
-      const TypePtr *adr_type = gvn.type(vec_adr)->isa_ptr();
-
-      vec_val_load = LoadVectorNode::make(0,
-                                          ctrl,
-                                          mem,
-                                          vec_adr,
-                                          adr_type,
-                                          num_elem,
-                                          bt);
-      vec_val_load = gvn.transform(vec_val_load);
-    }
+    assert (vec_val_load != NULL, "");
 
     C->set_max_vector_size(MAX2(C->max_vector_size(), vec_val_load->bottom_type()->is_vect()->length_in_bytes()));
     gvn.hash_delete(vec_unbox);

src/hotspot/share/opto/vector.hpp

@@ -29,15 +29,15 @@
 #include "opto/phaseX.hpp"
 #include "opto/type.hpp"
 #include "opto/vectornode.hpp"
+#include "ci/ciKlass.hpp"
 
 class PhaseVector : public Phase {
  private:
   PhaseIterGVN& _igvn;
 
   void expand_vbox_nodes();
   void expand_vbox_node(VectorBoxNode* vec_box);
-  Node* expand_vbox_node_helper(Node* vbox,
-                                Node* vbox_alloc,
+  Node* expand_vbox_node_helper(Node* vbox_alloc,
                                 Node* vect,
                                 const TypeInstPtr* box_type,
                                 const TypeVect* vect_type);

src/hotspot/share/opto/vectorIntrinsics.cpp

@@ -31,16 +31,24 @@
 #include "prims/vectorSupport.hpp"
 #include "runtime/stubRoutines.hpp"
 
-static bool is_vector(ciKlass* kls) { return VectorSupport::is_vector(kls); }
-static bool is_vector_mask(ciKlass* kls) { return VectorSupport::is_vector_mask(kls); }
-static bool is_vector_shuffle(ciKlass* kls) { return VectorSupport::is_vector_shuffle(kls); }
+static bool is_vector(ciKlass* klass) {
+  return klass->is_subclass_of(ciEnv::current()->vector_Vector_klass());
+}
+
+static bool is_vector_mask(ciKlass* klass) {
+  return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
+}
+
+static bool is_vector_shuffle(ciKlass* klass) {
+  return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
+}
 
 #ifdef ASSERT
 static bool check_vbox(const TypeInstPtr* vbox_type) {
   assert(vbox_type->klass_is_exact(), "");
 
   ciInstanceKlass* ik = vbox_type->instance_klass();
-  assert(VectorSupport::is_vector(ik), "not a vector");
+  assert(is_vector(ik), "not a vector");
 
   ciField* fd1 = ik->get_field_by_name(ciSymbols::ETYPE_name(), ciSymbols::class_signature(), /* is_static */ true);
   assert(fd1 != NULL, "element type info is missing");

src/hotspot/share/opto/vectornode.cpp

@@ -1633,15 +1633,15 @@ Node* VectorInsertNode::make(Node* vec, Node* new_val, int position) {
 
 Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) {
   Node* n = obj();
+  assert(n->is_InlineType(), "");
   // Vector APIs are lazily intrinsified, during parsing compiler emits a
   // call to intrinsic function, since most of the APIs return an abstract vector
   // hence a subsequent checkcast results into a graph shape comprising of CheckPP
   // and CheckCastPP chain. During lazy inline expansion, call gets replaced by
   // a VectorBox but we still need to traverse back through chain of cast nodes
   // to get to the VectorBox.
-  if (n->is_InlineType() &&
-      !n->is_VectorBox() &&
-      VectorSupport::is_vector(n->as_InlineType()->inline_klass())) {
+  if (!n->is_VectorBox() &&
+      VectorSupport::is_vector(n->as_InlineType()->inline_klass()->get_InlineKlass())) {
     n = n->as_InlineType()->get_oop();
   }
   n = n->uncast();
@@ -1695,7 +1695,7 @@ Node* VectorUnboxNode::Identity(PhaseGVN* phase) {
   // to get to the VectorBox.
   if (n->is_InlineType() &&
       !n->is_VectorBox() &&
-      VectorSupport::is_vector(n->as_InlineType()->inline_klass())) {
+      VectorSupport::is_vector(n->as_InlineType()->inline_klass()->get_InlineKlass())) {
     n = n->as_InlineType()->get_oop();
   }
   n = n->uncast();

src/hotspot/share/prims/vectorSupport.cpp

@@ -68,43 +68,22 @@ const char* VectorSupport::svmlname[VectorSupport::NUM_SVML_OP] = {
 };
 #endif
 
-
-bool VectorSupport::is_vector(ciKlass* klass) {
-  return klass->is_subclass_of(ciEnv::current()->vector_Vector_klass());
-}
-
 bool VectorSupport::is_vector(Klass* klass) {
   return klass->is_subclass_of(vmClasses::vector_Vector_klass());
 }
 
-bool VectorSupport::is_vector_payload_mf(ciKlass* klass) {
-  return klass->is_subclass_of(ciEnv::current()->vector_VectorPayloadMF_klass());
-}
-
 bool VectorSupport::is_vector_payload_mf(Klass* klass) {
   return klass->is_subclass_of(vmClasses::vector_VectorPayloadMF_klass());
 }
 
-bool VectorSupport::is_vector_mask(ciKlass* klass) {
-  return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
-}
-
 bool VectorSupport::is_vector_mask(Klass* klass) {
   return klass->is_subclass_of(vmClasses::vector_VectorMask_klass());
 }
 
-bool VectorSupport::is_vector_shuffle(ciKlass* klass) {
-  return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
-}
-
 bool VectorSupport::is_vector_shuffle(Klass* klass) {
   return klass->is_subclass_of(vmClasses::vector_VectorShuffle_klass());
 }
 
-bool VectorSupport::skip_value_scalarization(ciKlass* klass) {
-  return VectorSupport::is_vector(klass) || VectorSupport::is_vector_payload_mf(klass);
-}
-
 bool VectorSupport::skip_value_scalarization(Klass* klass) {
   return VectorSupport::is_vector(klass) || VectorSupport::is_vector_payload_mf(klass);
 }
@@ -399,7 +378,6 @@ instanceOop VectorSupport::allocate_vector(InstanceKlass* ik, frame* fr, Registe
   Handle vbox_h = Handle(THREAD, vbox);
 
   fieldDescriptor fd;
-  int elem_size = type2aelembytes(elem_bt);
   Symbol* payload_sig = VectorSupport::get_vector_payload_field_signature(elem_bt, num_elem);
   Klass* def = ik->find_field(vmSymbols::payload_name(), payload_sig, false, &fd);
   assert(def != NULL, "");

src/hotspot/share/prims/vectorSupport.hpp

@@ -31,7 +31,6 @@
 #include "oops/typeArrayOop.hpp"
 #include "runtime/registerMap.hpp"
 #include "utilities/exceptions.hpp"
-#include "ci/ciKlass.hpp"
 
 extern "C" {
   void JNICALL JVM_RegisterVectorSupportMethods(JNIEnv* env, jclass vsclass);
@@ -151,14 +150,9 @@ class VectorSupport : AllStatic {
   static Symbol* get_vector_payload_field_signature(BasicType elem_bt, int num_elem);
 
   static bool is_vector(Klass* klass);
-  static bool is_vector(ciKlass* klass);
   static bool is_vector_payload_mf(Klass* klass);
-  static bool is_vector_payload_mf(ciKlass* klass);
   static bool is_vector_mask(Klass* klass);
-  static bool is_vector_mask(ciKlass* klass);
   static bool is_vector_shuffle(Klass* klass);
-  static bool is_vector_shuffle(ciKlass* klass);
-  static bool skip_value_scalarization(ciKlass* klass);
   static bool skip_value_scalarization(Klass* klass);
 };
 #endif // SHARE_PRIMS_VECTORSUPPORT_HPP

src/hotspot/share/runtime/sharedRuntime.cpp

@@ -638,7 +638,7 @@ JRT_LEAF(address, SharedRuntime::exception_handler_for_return_address(JavaThread
   return raw_exception_handler_for_return_address(current, return_address);
 JRT_END
 
-JRT_LEAF(jint, SharedRuntime::is_vector_value_instance(InlineKlass* klass))
+JRT_LEAF(jint, SharedRuntime::skip_value_scalarization(InlineKlass* klass))
   return (jint)VectorSupport::skip_value_scalarization(klass);
 JRT_END
 

src/hotspot/share/runtime/sharedRuntime.hpp

@@ -185,7 +185,7 @@ class SharedRuntime: AllStatic {
   // exception handling across interpreter/compiler boundaries
   static address raw_exception_handler_for_return_address(JavaThread* current, address return_address);
   static address exception_handler_for_return_address(JavaThread* current, address return_address);
-  static int is_vector_value_instance(InlineKlass* klass);
+  static int skip_value_scalarization(InlineKlass* klass);
 
   // exception handling and implicit exceptions
   static address compute_compiled_exc_handler(CompiledMethod* nm, address ret_pc, Handle& exception,