8366668: [lworld] Follow-up work from the array metadata re-work

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

@@ -34,6 +34,7 @@
 #include "ci/ciArrayKlass.hpp"
 #include "ci/ciInlineKlass.hpp"
 #include "ci/ciInstance.hpp"
+#include "ci/ciObjArrayKlass.hpp"
 #include "code/compiledIC.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "gc/shared/gc_globals.hpp"
@@ -1394,6 +1395,7 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
   __ verify_oop(obj);
 
   if (op->fast_check()) {
+    assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
     // get object class
     // not a safepoint as obj null check happens earlier
     __ load_klass(rscratch1, obj);
@@ -1416,7 +1418,14 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
         // See if we get an immediate positive hit
         __ br(Assembler::EQ, *success_target);
         // check for self
-        __ cmp(klass_RInfo, k_RInfo);
+        if (k->is_loaded() && k->is_obj_array_klass()) {
+          // For a direct pointer comparison, we need the refined array klass pointer
+          ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
+          __ mov_metadata(rscratch1, k_refined->constant_encoding());
+          __ cmp(klass_RInfo, rscratch1);
+        } else {
+          __ cmp(klass_RInfo, k_RInfo);
+        }
         __ br(Assembler::EQ, *success_target);
 
         __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));

src/hotspot/cpu/aarch64/c1_LIRGenerator_aarch64.cpp

@@ -1204,7 +1204,7 @@ void LIRGenerator::do_NewObjectArray(NewObjectArray* x) {
   length.load_item_force(FrameMap::r19_opr);
   LIR_Opr len = length.result();
 
-  ciKlass* obj = ciArrayKlass::make(x->klass(), false, true, true);
+  ciKlass* obj = ciObjArrayKlass::make(x->klass());
 
   // TODO 8265122 Implement a fast path for this
   bool is_flat = obj->is_loaded() && obj->is_flat_array_klass();

src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp

@@ -1056,6 +1056,7 @@ OopMapSet* Runtime1::generate_code_for(StubId id, StubAssembler* sasm) {
         __ load_klass(obj, obj);
 
         // This is necessary because I am never in my own secondary_super list.
+        // TODO 8370341 Wouldn't this fail for arrays?
         __ cmp(obj, klass);
         __ br(Assembler::EQ, success);
 

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -1441,6 +1441,7 @@ void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
   // We move this check to the front of the fast path because many
   // type checks are in fact trivially successful in this manner,
   // so we get a nicely predicted branch right at the start of the check.
+  // TODO 8370341 For a direct pointer comparison, we need the refined array klass pointer
   cmp(sub_klass, super_klass);
   br(Assembler::EQ, *L_success);
 

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

@@ -33,6 +33,7 @@
 #include "ci/ciArrayKlass.hpp"
 #include "ci/ciInlineKlass.hpp"
 #include "ci/ciInstance.hpp"
+#include "ci/ciObjArrayKlass.hpp"
 #include "compiler/oopMap.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "gc/shared/gc_globals.hpp"
@@ -1406,7 +1407,7 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
   __ verify_oop(obj);
 
   if (op->fast_check()) {
-    // TODO 8366668 Is this correct? I don't think so. Probably we now always go to the slow path here. Same on AArch64.
+    assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
     // get object class
     // not a safepoint as obj null check happens earlier
     if (UseCompressedClassPointers) {
@@ -1431,7 +1432,14 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
         // See if we get an immediate positive hit
         __ jcc(Assembler::equal, *success_target);
         // check for self
-        __ cmpptr(klass_RInfo, k_RInfo);
+        if (k->is_loaded() && k->is_obj_array_klass()) {
+          // For a direct pointer comparison, we need the refined array klass pointer
+          ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
+          __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
+          __ cmpptr(klass_RInfo, tmp_load_klass);
+        } else {
+          __ cmpptr(klass_RInfo, k_RInfo);
+        }
         __ jcc(Assembler::equal, *success_target);
 
         __ push_ppx(klass_RInfo);

src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp

@@ -1215,7 +1215,7 @@ void LIRGenerator::do_NewObjectArray(NewObjectArray* x) {
   length.load_item_force(FrameMap::rbx_opr);
   LIR_Opr len = length.result();
 
-  ciKlass* obj = ciArrayKlass::make(x->klass(), false, true, true);
+  ciKlass* obj = ciObjArrayKlass::make(x->klass());
 
   // TODO 8265122 Implement a fast path for this
   bool is_flat = obj->is_loaded() && obj->is_flat_array_klass();

src/hotspot/cpu/x86/c1_Runtime1_x86.cpp

@@ -1250,6 +1250,7 @@ OopMapSet* Runtime1::generate_code_for(StubId id, StubAssembler* sasm) {
         __ load_klass(obj, obj, /*tmp*/temp1);
 
         // This is necessary because I am never in my own secondary_super list.
+        // TODO 8370341 Wouldn't this fail for arrays?
         __ cmpptr(obj, klass);
         __ jcc(Assembler::equal, same);
 

src/hotspot/cpu/x86/macroAssembler_x86.cpp

@@ -4178,6 +4178,7 @@ void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
   // We move this check to the front of the fast path because many
   // type checks are in fact trivially successful in this manner,
   // so we get a nicely predicted branch right at the start of the check.
+  // TODO 8370341 For a direct pointer comparison, we need the refined array klass pointer
   cmpptr(sub_klass, super_klass);
   local_jcc(Assembler::equal, *L_success);
 

src/hotspot/share/c1/c1_LIRGenerator.cpp

@@ -771,6 +771,11 @@ void LIRGenerator::arraycopy_helper(Intrinsic* x, int* flagsp, ciArrayKlass** ex
     if (expected_type == nullptr) expected_type = src_declared_type;
     if (expected_type == nullptr) expected_type = dst_declared_type;
 
+    if (expected_type != nullptr && expected_type->is_obj_array_klass()) {
+      // For a direct pointer comparison, we need the refined array klass pointer
+      expected_type = ciObjArrayKlass::make(expected_type->as_array_klass()->element_klass());
+    }
+
     src_objarray = (src_exact_type && src_exact_type->is_obj_array_klass()) || (src_declared_type && src_declared_type->is_obj_array_klass());
     dst_objarray = (dst_exact_type && dst_exact_type->is_obj_array_klass()) || (dst_declared_type && dst_declared_type->is_obj_array_klass());
   }
@@ -896,12 +901,6 @@ void LIRGenerator::arraycopy_helper(Intrinsic* x, int* flagsp, ciArrayKlass** ex
     }
   }
   *flagsp = flags;
-
-  // TODO 8366668
-  if (expected_type != nullptr && expected_type->is_obj_array_klass()) {
-    expected_type = ciArrayKlass::make(expected_type->as_array_klass()->element_klass(), false, true, true);
-  }
-
   *expected_typep = (ciArrayKlass*)expected_type;
 }
 
@@ -1901,6 +1900,11 @@ void LIRGenerator::do_StoreIndexed(StoreIndexed* x) {
     }
   }
 
+  if (GenerateArrayStoreCheck && needs_store_check) {
+    CodeEmitInfo* store_check_info = new CodeEmitInfo(range_check_info);
+    array_store_check(value.result(), array.result(), store_check_info, x->profiled_method(), x->profiled_bci());
+  }
+
   if (x->should_profile()) {
     if (is_loaded_flat_array) {
       // No need to profile a store to a flat array of known type. This can happen if
@@ -1916,16 +1920,11 @@ void LIRGenerator::do_StoreIndexed(StoreIndexed* x) {
       profile_array_type(x, md, store_data);
       assert(store_data->is_ArrayStoreData(), "incorrect profiling entry");
       if (x->array()->maybe_null_free_array()) {
-        profile_null_free_array(array, md, store_data);
+        profile_null_free_array(array, md, data);
       }
     }
   }
 
-  if (GenerateArrayStoreCheck && needs_store_check) {
-    CodeEmitInfo* store_check_info = new CodeEmitInfo(range_check_info);
-    array_store_check(value.result(), array.result(), store_check_info, x->profiled_method(), x->profiled_bci());
-  }
-
   if (is_loaded_flat_array) {
     // TODO 8350865 This is currently dead code
     if (!x->value()->is_null_free()) {
@@ -2281,7 +2280,7 @@ void LIRGenerator::do_LoadIndexed(LoadIndexed* x) {
   }
 
   ciMethodData* md = nullptr;
-  ciArrayLoadData* load_data = nullptr;
+  ciProfileData* data = nullptr;
   if (x->should_profile()) {
     if (x->array()->is_loaded_flat_array()) {
       // No need to profile a load from a flat array of known type. This can happen if
@@ -2291,9 +2290,9 @@ void LIRGenerator::do_LoadIndexed(LoadIndexed* x) {
       int bci = x->profiled_bci();
       md = x->profiled_method()->method_data();
       assert(md != nullptr, "Sanity");
-      ciProfileData* data = md->bci_to_data(bci);
+      data = md->bci_to_data(bci);
       assert(data != nullptr && data->is_ArrayLoadData(), "incorrect profiling entry");
-      load_data = (ciArrayLoadData*)data;
+      ciArrayLoadData* load_data = (ciArrayLoadData*)data;
       profile_array_type(x, md, load_data);
     }
   }
@@ -2317,7 +2316,7 @@ void LIRGenerator::do_LoadIndexed(LoadIndexed* x) {
     LoadFlattenedArrayStub* slow_path = nullptr;
 
     if (x->should_profile() && x->array()->maybe_null_free_array()) {
-      profile_null_free_array(array, md, load_data);
+      profile_null_free_array(array, md, data);
     }
 
     if (x->elt_type() == T_OBJECT && x->array()->maybe_flat_array()) {
@@ -2343,7 +2342,7 @@ void LIRGenerator::do_LoadIndexed(LoadIndexed* x) {
   }
 
   if (x->should_profile()) {
-    profile_element_type(element, md, load_data);
+    profile_element_type(element, md, (ciArrayLoadData*)data);
   }
 }
 
@@ -2819,16 +2818,6 @@ ciKlass* LIRGenerator::profile_type(ciMethodData* md, int md_base_offset, int md
     assert(type == nullptr || type->is_klass(), "type should be class");
     exact_klass = (type != nullptr && type->is_loaded()) ? (ciKlass*)type : nullptr;
 
-    // TODO 8366668
-    if (exact_klass != nullptr && exact_klass->is_obj_array_klass()) {
-      if (exact_klass->as_obj_array_klass()->element_klass()->is_inlinetype()) {
-        // Could be flat, null free etc.
-        exact_klass = nullptr;
-      } else {
-        exact_klass = ciObjArrayKlass::make(exact_klass->as_array_klass()->element_klass(), true);
-      }
-    }
-
     do_update = exact_klass == nullptr || ciTypeEntries::valid_ciklass(profiled_k) != exact_klass;
   }
 
@@ -2866,20 +2855,21 @@ ciKlass* LIRGenerator::profile_type(ciMethodData* md, int md_base_offset, int md
         exact_klass = exact_signature_k;
       }
     }
-
-    // TODO 8366668
-    if (exact_klass != nullptr && exact_klass->is_obj_array_klass()) {
-      if (exact_klass->as_obj_array_klass()->element_klass()->is_inlinetype()) {
-        // Could be flat, null free etc.
-        exact_klass = nullptr;
-      } else {
-        exact_klass = ciObjArrayKlass::make(exact_klass->as_array_klass()->element_klass(), true);
-      }
-    }
-
     do_update = exact_klass == nullptr || ciTypeEntries::valid_ciklass(profiled_k) != exact_klass;
   }
 
+  if (exact_klass != nullptr && exact_klass->is_obj_array_klass()) {
+    if (exact_klass->can_be_inline_array_klass()) {
+      // Inline type arrays can have additional properties, we need to load the klass
+      // TODO 8350865 Can we do better here and track the properties?
+      exact_klass = nullptr;
+      do_update = true;
+    } else {
+      // For a direct pointer comparison, we need the refined array klass pointer
+      exact_klass = ciObjArrayKlass::make(exact_klass->as_array_klass()->element_klass());
+      do_update = ciTypeEntries::valid_ciklass(profiled_k) != exact_klass;
+    }
+  }
   if (!do_null && !do_update) {
     return result;
   }
@@ -2940,22 +2930,24 @@ void LIRGenerator::profile_flags(ciMethodData* md, ciProfileData* data, int flag
   LIR_Address* addr = new LIR_Address(mdp, md->byte_offset_of_slot(data, DataLayout::flags_offset()), T_BYTE);
   LIR_Opr flags = new_register(T_INT);
   __ move(addr, flags);
+  LIR_Opr update;
   if (condition != lir_cond_always) {
-    LIR_Opr update = new_register(T_INT);
+    update = new_register(T_INT);
     __ cmove(condition, LIR_OprFact::intConst(0), LIR_OprFact::intConst(flag), update, T_INT);
   } else {
-    __ logical_or(flags, LIR_OprFact::intConst(flag), flags);
+    update = LIR_OprFact::intConst(flag);
   }
+  __ logical_or(flags, update, flags);
   __ store(flags, addr);
 }
 
-template <class ArrayData> void LIRGenerator::profile_null_free_array(LIRItem array, ciMethodData* md, ArrayData* load_store) {
+void LIRGenerator::profile_null_free_array(LIRItem array, ciMethodData* md, ciProfileData* data) {
   assert(compilation()->profile_array_accesses(), "array access profiling is disabled");
   LabelObj* L_end = new LabelObj();
   LIR_Opr tmp = new_register(T_METADATA);
   __ check_null_free_array(array.result(), tmp);
 
-  profile_flags(md, load_store, ArrayStoreData::null_free_array_byte_constant(), lir_cond_equal);
+  profile_flags(md, data, ArrayStoreData::null_free_array_byte_constant(), lir_cond_equal);
 }
 
 template <class ArrayData> void LIRGenerator::profile_array_type(AccessIndexed* x, ciMethodData*& md, ArrayData*& load_store) {

src/hotspot/share/c1/c1_LIRGenerator.hpp

@@ -493,7 +493,7 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
   void profile_parameters(Base* x);
   void profile_parameters_at_call(ProfileCall* x);
   void profile_flags(ciMethodData* md, ciProfileData* load_store, int flag, LIR_Condition condition = lir_cond_always);
-  template <class ArrayData> void profile_null_free_array(LIRItem array, ciMethodData* md, ArrayData* load_store);
+  void profile_null_free_array(LIRItem array, ciMethodData* md, ciProfileData* load_store);
   template <class ArrayData> void profile_array_type(AccessIndexed* x, ciMethodData*& md, ArrayData*& load_store);
   void profile_element_type(Value element, ciMethodData* md, ciArrayLoadData* load_store);
   bool profile_inline_klass(ciMethodData* md, ciProfileData* data, Value value, int flag);

src/hotspot/share/ci/ciArrayKlass.cpp

@@ -104,34 +104,9 @@ bool ciArrayKlass::is_leaf_type() {
 ciArrayKlass* ciArrayKlass::make(ciType* element_type, bool null_free, bool atomic, bool vm_type) {
   if (element_type->is_primitive_type()) {
     return ciTypeArrayKlass::make(element_type->basic_type());
+  } else {
+    return ciObjArrayKlass::make(element_type->as_klass(), vm_type, null_free, atomic);
   }
-
-  ciKlass* klass = element_type->as_klass();
-  assert(!null_free || !klass->is_loaded() || klass->is_inlinetype() || klass->is_abstract() ||
-         klass->is_java_lang_Object(), "only value classes are null free");
-  if (klass->is_loaded() && klass->is_inlinetype() && vm_type) {
-    GUARDED_VM_ENTRY(
-      EXCEPTION_CONTEXT;
-      InlineKlass* vk = InlineKlass::cast(klass->get_Klass());
-      ArrayKlass::ArrayProperties props = ArrayKlass::ArrayProperties::DEFAULT;
-      if (null_free) {
-        props = (ArrayKlass::ArrayProperties)(props | ArrayKlass::ArrayProperties::NULL_RESTRICTED);
-      }
-      if (!atomic) {
-        props = (ArrayKlass::ArrayProperties)(props | ArrayKlass::ArrayProperties::NON_ATOMIC);
-      }
-      ArrayKlass* ak = vk->array_klass(THREAD);
-      ak = ObjArrayKlass::cast(ak)->klass_with_properties(props, THREAD);
-      if (HAS_PENDING_EXCEPTION) {
-        CLEAR_PENDING_EXCEPTION;
-      } else if (ak->is_flatArray_klass()) {
-        return CURRENT_THREAD_ENV->get_flat_array_klass(ak);
-      } else if (ak->is_refArray_klass()) {
-        return CURRENT_THREAD_ENV->get_obj_array_klass(ak);
-      }
-    )
-  }
-  return ciObjArrayKlass::make(klass, vm_type);
 }
 
 int ciArrayKlass::array_header_in_bytes() {

src/hotspot/share/ci/ciMethod.cpp

@@ -674,6 +674,14 @@ bool ciMethod::array_access_profiled_type(int bci, ciKlass*& array_type, ciKlass
         element_ptr = array_access->element()->ptr_kind();
         flat_array = array_access->flat_array();
         null_free_array = array_access->null_free_array();
+#ifdef ASSERT
+        if (array_type != nullptr) {
+          bool flat = array_type->is_flat_array_klass();
+          bool null_free = array_type->as_array_klass()->is_elem_null_free();
+          assert(!flat || flat_array, "inconsistency");
+          assert(!null_free || null_free_array, "inconsistency");
+        }
+#endif
         return true;
       } else if (data->is_ArrayStoreData()) {
         ciArrayStoreData* array_access = (ciArrayStoreData*) data->as_ArrayStoreData();
@@ -693,6 +701,14 @@ bool ciMethod::array_access_profiled_type(int bci, ciKlass*& array_type, ciKlass
         } else {
           element_ptr = ProfileMaybeNull;
         }
+#ifdef ASSERT
+        if (array_type != nullptr) {
+          bool flat = array_type->is_flat_array_klass();
+          bool null_free = array_type->as_array_klass()->is_elem_null_free();
+          assert(!flat || flat_array, "inconsistency");
+          assert(!null_free || null_free_array, "inconsistency");
+        }
+#endif
         return true;
       }
     }

src/hotspot/share/ci/ciMethodData.cpp

@@ -997,6 +997,7 @@ void ciSpeculativeTrapData::print_data_on(outputStream* st, const char* extra) c
 
 void ciArrayStoreData::print_data_on(outputStream* st, const char* extra) const {
   print_shared(st, "ciArrayStoreData", extra);
+  st->cr();
   tab(st, true);
   st->print("array");
   array()->print_data_on(st);
@@ -1007,6 +1008,7 @@ void ciArrayStoreData::print_data_on(outputStream* st, const char* extra) const
 
 void ciArrayLoadData::print_data_on(outputStream* st, const char* extra) const {
   print_shared(st, "ciArrayLoadData", extra);
+  st->cr();
   tab(st, true);
   st->print("array");
   array()->print_data_on(st);