8364191: [lworld] Accesses to atomic flat fields prevent scalar replacement

src/hotspot/share/opto/classes.hpp

@@ -387,6 +387,8 @@ macro(URShiftL)
 macro(XorI)
 macro(XorL)
 macro(InlineType)
+macro(LoadFlat)
+macro(StoreFlat)
 macro(Vector)
 macro(AddVB)
 macro(AddVS)

src/hotspot/share/opto/compile.cpp

@@ -410,6 +410,9 @@ void Compile::remove_useless_node(Node* dead) {
   if (dead->is_InlineType()) {
     remove_inline_type(dead);
   }
+  if (dead->is_LoadFlat() || dead->is_StoreFlat()) {
+    remove_flat_access(dead);
+  }
   if (dead->for_merge_stores_igvn()) {
     remove_from_merge_stores_igvn(dead);
   }
@@ -469,6 +472,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis
   remove_useless_nodes(_expensive_nodes,    useful); // remove useless expensive nodes
   remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
   remove_useless_nodes(_inline_type_nodes,  useful); // remove useless inline type nodes
+  remove_useless_nodes(_flat_access_nodes, useful);  // remove useless flat access nodes
 #ifdef ASSERT
   if (_modified_nodes != nullptr) {
     _modified_nodes->remove_useless_nodes(useful.member_set());
@@ -681,6 +685,7 @@ Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci,
       _expensive_nodes(comp_arena(), 8, 0, nullptr),
       _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
       _inline_type_nodes (comp_arena(), 8, 0, nullptr),
+      _flat_access_nodes(comp_arena(), 8, 0, nullptr),
       _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
       _unstable_if_traps(comp_arena(), 8, 0, nullptr),
       _coarsened_locks(comp_arena(), 8, 0, nullptr),
@@ -2093,6 +2098,34 @@ void Compile::process_inline_types(PhaseIterGVN &igvn, bool remove) {
   igvn.optimize();
 }
 
+void Compile::add_flat_access(Node* n) {
+  assert(n != nullptr && (n->Opcode() == Op_LoadFlat || n->Opcode() == Op_StoreFlat), "unexpected node %s", n == nullptr ? "nullptr" : n->Name());
+  assert(!_flat_access_nodes.contains(n), "duplicate insertion");
+  _flat_access_nodes.push(n);
+}
+
+void Compile::remove_flat_access(Node* n) {
+  assert(n != nullptr && (n->Opcode() == Op_LoadFlat || n->Opcode() == Op_StoreFlat), "unexpected node %s", n == nullptr ? "nullptr" : n->Name());
+  _flat_access_nodes.remove_if_existing(n);
+}
+
+void Compile::process_flat_accesses(PhaseIterGVN& igvn) {
+  assert(igvn._worklist.size() == 0, "should be empty");
+  igvn.set_delay_transform(true);
+  for (int i = _flat_access_nodes.length() - 1; i >= 0; i--) {
+    Node* n = _flat_access_nodes.at(i);
+    assert(n != nullptr, "unexpected nullptr");
+    if (n->is_LoadFlat()) {
+      n->as_LoadFlat()->expand_atomic(igvn);
+    } else {
+      n->as_StoreFlat()->expand_atomic(igvn);
+    }
+  }
+  _flat_access_nodes.clear_and_deallocate();
+  igvn.set_delay_transform(false);
+  igvn.optimize();
+}
+
 void Compile::adjust_flat_array_access_aliases(PhaseIterGVN& igvn) {
   if (!_has_flat_accesses) {
     return;
@@ -2942,6 +2975,11 @@ void Compile::Optimize() {
     } while (progress);
   }
 
+  process_flat_accesses(igvn);
+  if (failing()) {
+    return;
+  }
+
   // Loop transforms on the ideal graph.  Range Check Elimination,
   // peeling, unrolling, etc.
 

src/hotspot/share/opto/compile.hpp

@@ -45,6 +45,7 @@
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/timerTrace.hpp"
 #include "runtime/vmThread.hpp"
+#include "utilities/growableArray.hpp"
 #include "utilities/ticks.hpp"
 #include "utilities/vmEnums.hpp"
 
@@ -385,6 +386,7 @@ class Compile : public Phase {
   GrowableArray<Node*>  _expensive_nodes;       // List of nodes that are expensive to compute and that we'd better not let the GVN freely common
   GrowableArray<Node*>  _for_post_loop_igvn;    // List of nodes for IGVN after loop opts are over
   GrowableArray<Node*>  _inline_type_nodes;     // List of InlineType nodes
+  GrowableArray<Node*>  _flat_access_nodes;     // List of LoadFlat and StoreFlat nodes
   GrowableArray<Node*>  _for_merge_stores_igvn; // List of nodes for IGVN merge stores
   GrowableArray<UnstableIfTrap*> _unstable_if_traps;        // List of ifnodes after IGVN
   GrowableArray<Node_List*> _coarsened_locks;   // List of coarsened Lock and Unlock nodes
@@ -796,6 +798,17 @@ class Compile : public Phase {
   void remove_inline_type(Node* n);
   void process_inline_types(PhaseIterGVN &igvn, bool remove = false);
 
+  void add_flat_access(Node* n);
+  void remove_flat_access(Node* n);
+  void process_flat_accesses(PhaseIterGVN& igvn);
+
+  template <class F>
+  void for_each_flat_access(F consumer) {
+    for (int i = _flat_access_nodes.length() - 1; i >= 0; i--) {
+      consumer(_flat_access_nodes.at(i));
+    }
+  }
+
   void adjust_flat_array_access_aliases(PhaseIterGVN& igvn);
 
   void record_unstable_if_trap(UnstableIfTrap* trap);

src/hotspot/share/opto/escape.cpp

@@ -425,7 +425,15 @@ bool ConnectionGraph::compute_escape() {
 #endif
   }
 
-  // 6. Reduce allocation merges used as debug information. This is done after
+  // 6. Expand flat accesses if the object does not escape. This adds nodes to
+  // the graph, so it has to be after split_unique_types. This expands atomic
+  // mismatched accesses (though encapsulated in LoadFlats and StoreFlats) into
+  // non-mismatched accesses, so it is better before reduce allocation merges.
+  if (has_non_escaping_obj) {
+    optimize_flat_accesses(sfn_worklist);
+  }
+
+  // 7. Reduce allocation merges used as debug information. This is done after
   // split_unique_types because the methods used to create SafePointScalarObject
   // need to traverse the memory graph to find values for object fields. We also
   // set to null the scalarized inputs of reducible Phis so that the Allocate
@@ -1679,13 +1687,24 @@ void ConnectionGraph::add_node_to_connection_graph(Node *n, Unique_Node_List *de
       }
       break;
     }
+    case Op_LoadFlat:
+      // Treat LoadFlat similar to an unknown call that receives nothing and produces its results
+      map_ideal_node(n, phantom_obj);
+      break;
+    case Op_StoreFlat:
+      // Treat StoreFlat similar to a call that escapes the stored flattened fields
+      delayed_worklist->push(n);
+      break;
     case Op_Proj: {
       // we are only interested in the oop result projection from a call
       if (n->as_Proj()->_con >= TypeFunc::Parms && n->in(0)->is_Call() &&
           (n->in(0)->as_Call()->returns_pointer() || n->bottom_type()->isa_ptr())) {
         assert((n->as_Proj()->_con == TypeFunc::Parms && n->in(0)->as_Call()->returns_pointer()) ||
                n->in(0)->as_Call()->tf()->returns_inline_type_as_fields(), "what kind of oop return is it?");
         add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), delayed_worklist);
+      } else if (n->as_Proj()->_con >= TypeFunc::Parms && n->in(0)->is_LoadFlat() && igvn->type(n)->isa_ptr()) {
+        // Treat LoadFlat outputs similar to a call return value
+        add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), delayed_worklist);
       }
       break;
     }
@@ -1770,7 +1789,7 @@ void ConnectionGraph::add_final_edges(Node *n) {
     process_call_arguments(n->as_Call());
     return;
   }
-  assert(n->is_Store() || n->is_LoadStore() ||
+  assert(n->is_Store() || n->is_LoadStore() || n->is_StoreFlat() ||
          ((n_ptn != nullptr) && (n_ptn->ideal_node() != nullptr)),
          "node should be registered already");
   int opcode = n->Opcode();
@@ -1839,11 +1858,32 @@ void ConnectionGraph::add_final_edges(Node *n) {
       }
       break;
     }
+    case Op_StoreFlat: {
+      // StoreFlat globally escapes its stored flattened fields
+      InlineTypeNode* value = n->as_StoreFlat()->value();
+      ciInlineKlass* vk = _igvn->type(value)->inline_klass();
+      for (int i = 0; i < vk->nof_nonstatic_fields(); i++) {
+        ciField* field = vk->nonstatic_field_at(i);
+        if (field->type()->is_primitive_type()) {
+          continue;
+        }
+
+        Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true);
+        PointsToNode* field_value_ptn = ptnode_adr(field_value->_idx);
+        set_escape_state(field_value_ptn, PointsToNode::GlobalEscape NOT_PRODUCT(COMMA "store into a flat field"));
+      }
+      break;
+    }
     case Op_Proj: {
-      // we are only interested in the oop result projection from a call
-      assert((n->as_Proj()->_con == TypeFunc::Parms && n->in(0)->as_Call()->returns_pointer()) ||
-             n->in(0)->as_Call()->tf()->returns_inline_type_as_fields(), "what kind of oop return is it?");
-      add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), nullptr);
+      if (n->in(0)->is_Call()) {
+        // we are only interested in the oop result projection from a call
+        assert((n->as_Proj()->_con == TypeFunc::Parms && n->in(0)->as_Call()->returns_pointer()) ||
+              n->in(0)->as_Call()->tf()->returns_inline_type_as_fields(), "what kind of oop return is it?");
+        add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), nullptr);
+      } else if (n->in(0)->is_LoadFlat()) {
+        // Treat LoadFlat outputs similar to a call return value
+        add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), nullptr);
+      }
       break;
     }
     case Op_Rethrow: // Exception object escapes
@@ -3349,6 +3389,31 @@ void ConnectionGraph::optimize_ideal_graph(GrowableArray<Node*>& ptr_cmp_worklis
   }
 }
 
+// Atomic flat accesses on non-escaping objects can be optimized to non-atomic accesses
+void ConnectionGraph::optimize_flat_accesses(GrowableArray<SafePointNode*>& sfn_worklist) {
+  PhaseIterGVN& igvn = *_igvn;
+  bool delay = igvn.delay_transform();
+  igvn.set_delay_transform(true);
+  igvn.C->for_each_flat_access([&](Node* n) {
+    Node* base = n->is_LoadFlat() ? n->as_LoadFlat()->base() : n->as_StoreFlat()->base();
+    if (!not_global_escape(base)) {
+      return;
+    }
+
+    bool expanded;
+    if (n->is_LoadFlat()) {
+      expanded = n->as_LoadFlat()->expand_non_atomic(igvn);
+    } else {
+      expanded = n->as_StoreFlat()->expand_non_atomic(igvn);
+    }
+    if (expanded) {
+      sfn_worklist.remove(n->as_SafePoint());
+      igvn.C->remove_flat_access(n);
+    }
+  });
+  igvn.set_delay_transform(delay);
+}
+
 // Optimize objects compare.
 const TypeInt* ConnectionGraph::optimize_ptr_compare(Node* left, Node* right) {
   const TypeInt* UNKNOWN = TypeInt::CC;    // [-1, 0,1]

src/hotspot/share/opto/escape.hpp

@@ -481,6 +481,8 @@ class ConnectionGraph: public ArenaObj {
   // Optimize ideal graph.
   void optimize_ideal_graph(GrowableArray<Node*>& ptr_cmp_worklist,
                             GrowableArray<MemBarStoreStoreNode*>& storestore_worklist);
+  // Expand flat accesses to accesses to each component if the object does not escape
+  void optimize_flat_accesses(GrowableArray<SafePointNode*>& sfn_worklist);
   // Optimize objects compare.
   const TypeInt* optimize_ptr_compare(Node* left, Node* right);
 

src/hotspot/share/opto/graphKit.cpp

@@ -4650,6 +4650,7 @@ void GraphKit::add_parse_predicates(int nargs) {
 }
 
 void GraphKit::sync_kit(IdealKit& ideal) {
+  reset_memory();
   set_all_memory(ideal.merged_memory());
   set_i_o(ideal.i_o());
   set_control(ideal.ctrl());

src/hotspot/share/opto/idealKit.cpp

@@ -311,7 +311,9 @@ Node* IdealKit::delay_transform(Node* n) {
 //-----------------------------new_cvstate-----------------------------------
 Node* IdealKit::new_cvstate() {
   uint sz = _var_ct + first_var;
-  return new Node(sz);
+  Node* state = new Node(sz);
+  C->record_for_igvn(state);
+  return state;
 }
 
 //-----------------------------copy_cvstate-----------------------------------