From 443cef014b5d45f87835941477169bfac4498a52 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Wed, 12 Mar 2025 13:58:05 -0400 Subject: [PATCH 01/15] add const where possible --- src/hotspot/share/opto/loopnode.cpp | 43 ++++++++++++++--------------- src/hotspot/share/opto/loopnode.hpp | 23 ++++++++------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 7d83d602ac9d8..02c3d4ea19ab6 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -310,7 +310,7 @@ IdealLoopTree* PhaseIdealLoop::insert_outer_loop(IdealLoopTree* loop, LoopNode* // clone the outer loop as well as the inner, unrolling needs to only // clone the inner loop etc. No optimizations need to change the outer // strip mined loop as it is only a skeleton. -IdealLoopTree* PhaseIdealLoop::create_outer_strip_mined_loop(BoolNode *test, Node *cmp, Node *init_control, +IdealLoopTree* PhaseIdealLoop::create_outer_strip_mined_loop(Node *init_control, IdealLoopTree* loop, float cl_prob, float le_fcnt, Node*& entry_control, Node*& iffalse) { Node* outer_test = intcon(0); @@ -342,7 +342,7 @@ IdealLoopTree* PhaseIdealLoop::create_outer_strip_mined_loop(BoolNode *test, Nod return outer_ilt; } -void PhaseIdealLoop::insert_loop_limit_check_predicate(ParsePredicateSuccessProj* loop_limit_check_parse_proj, +void PhaseIdealLoop::insert_loop_limit_check_predicate(const ParsePredicateSuccessProj* loop_limit_check_parse_proj, Node* cmp_limit, Node* bol) { assert(loop_limit_check_parse_proj->in(0)->is_ParsePredicate(), "must be parse predicate"); Node* new_predicate_proj = create_new_if_for_predicate(loop_limit_check_parse_proj, nullptr, @@ -364,7 +364,7 @@ void PhaseIdealLoop::insert_loop_limit_check_predicate(ParsePredicateSuccessProj #endif } -Node* PhaseIdealLoop::loop_exit_control(Node* x, IdealLoopTree* loop) { +Node* PhaseIdealLoop::loop_exit_control(const Node* x, const IdealLoopTree* loop) const { // Counted loop head must be a good RegionNode with only 3 not null // control input edges: Self, Entry, LoopBack. if (x->in(LoopNode::Self) == nullptr || x->req() != 3 || loop->_irreducible) { @@ -404,8 +404,8 @@ Node* PhaseIdealLoop::loop_exit_control(Node* x, IdealLoopTree* loop) { return iftrue; } -Node* PhaseIdealLoop::loop_exit_test(Node* back_control, IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob) { - Node* iftrue = back_control; +Node* PhaseIdealLoop::loop_exit_test(const Node* back_control, const IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob) { + const Node* iftrue = back_control; uint iftrue_op = iftrue->Opcode(); Node* iff = iftrue->in(0); BoolNode* test = iff->in(1)->as_Bool(); @@ -444,7 +444,7 @@ Node* PhaseIdealLoop::loop_exit_test(Node* back_control, IdealLoopTree* loop, No return cmp; } -Node* PhaseIdealLoop::loop_iv_incr(Node* incr, Node* x, IdealLoopTree* loop, Node*& phi_incr) { +Node* PhaseIdealLoop::loop_iv_incr(Node* incr, const Node* x, const IdealLoopTree* loop, const Node*& phi_incr) { if (incr->is_Phi()) { if (incr->as_Phi()->region() != x || incr->req() != 3) { return nullptr; // Not simple trip counter expression @@ -458,7 +458,7 @@ Node* PhaseIdealLoop::loop_iv_incr(Node* incr, Node* x, IdealLoopTree* loop, Nod return incr; } -Node* PhaseIdealLoop::loop_iv_stride(Node* incr, IdealLoopTree* loop, Node*& xphi) { +Node* PhaseIdealLoop::loop_iv_stride(const Node* incr, Node*& xphi) { assert(incr->Opcode() == Op_AddI || incr->Opcode() == Op_AddL, "caller resp."); // Get merge point xphi = incr->in(1); @@ -467,14 +467,13 @@ Node* PhaseIdealLoop::loop_iv_stride(Node* incr, IdealLoopTree* loop, Node*& xph if (!xphi->is_Con()) { // Is the other guy a constant? return nullptr; // Nope, unknown stride, bail out } - Node *tmp = xphi; // 'incr' is commutative, so ok to swap - xphi = stride; - stride = tmp; + + swap(xphi, stride); // 'incr' is commutative, so ok to swap } return stride; } -PhiNode* PhaseIdealLoop::loop_iv_phi(Node* xphi, Node* phi_incr, Node* x, IdealLoopTree* loop) { +PhiNode* PhaseIdealLoop::loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x) { if (!xphi->is_Phi()) { return nullptr; // Too much math on the trip counter } @@ -615,7 +614,7 @@ void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Nod // Find a safepoint node that dominates the back edge. We need a // SafePointNode so we can use its jvm state to create empty // predicates. -static bool no_side_effect_since_safepoint(Compile* C, Node* x, Node* mem, MergeMemNode* mm, PhaseIdealLoop* phase) { +static bool no_side_effect_since_safepoint(Compile* C, const Node* x, const Node* mem, MergeMemNode* mm, const PhaseIdealLoop* phase) { SafePointNode* safepoint = nullptr; for (DUIterator_Fast imax, i = x->fast_outs(imax); i < imax; i++) { Node* u = x->fast_out(i); @@ -668,7 +667,7 @@ static bool no_side_effect_since_safepoint(Compile* C, Node* x, Node* mem, Merge return true; } -SafePointNode* PhaseIdealLoop::find_safepoint(Node* back_control, Node* x, IdealLoopTree* loop) { +SafePointNode* PhaseIdealLoop::find_safepoint(Node* back_control, const Node* x, const IdealLoopTree* loop) { IfNode* exit_test = back_control->in(0)->as_If(); SafePointNode* safepoint = nullptr; if (exit_test->in(0)->is_SafePoint() && exit_test->in(0)->outcnt() == 1) { @@ -1476,16 +1475,16 @@ void PhaseIdealLoop::check_counted_loop_shape(IdealLoopTree* loop, Node* x, Basi Node* cmp = loop_exit_test(back_control, loop, incr, limit, mask, cl_prob); assert(cmp != nullptr && cmp->Opcode() == Op_Cmp(bt), "no exit test"); - Node* phi_incr = nullptr; + const Node* phi_incr = nullptr; incr = loop_iv_incr(incr, x, loop, phi_incr); assert(incr != nullptr && incr->Opcode() == Op_Add(bt), "no incr"); Node* xphi = nullptr; - Node* stride = loop_iv_stride(incr, loop, xphi); + Node* stride = loop_iv_stride(incr, xphi); assert(stride != nullptr, "no stride"); - PhiNode* phi = loop_iv_phi(xphi, phi_incr, x, loop); + PhiNode* phi = loop_iv_phi(xphi, phi_incr, x); assert(phi != nullptr && phi->in(LoopNode::LoopBackControl) == incr, "No phi"); @@ -1505,7 +1504,7 @@ void PhaseIdealLoop::check_counted_loop_shape(IdealLoopTree* loop, Node* x, Basi #ifdef ASSERT // convert an int counted loop to a long counted to stress handling of // long counted loops -bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, Node* phi, IdealLoopTree* loop) { +bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, const Node* phi, const IdealLoopTree* loop) { Unique_Node_List iv_nodes; Node_List old_new; iv_nodes.push(cmp); @@ -1634,7 +1633,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ incr = incr->in(1); } - Node* phi_incr = nullptr; + const Node* phi_incr = nullptr; incr = loop_iv_incr(incr, x, loop, phi_incr); if (incr == nullptr) { return false; @@ -1650,7 +1649,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ assert(incr->Opcode() == Op_Add(iv_bt), "wrong increment code"); Node* xphi = nullptr; - Node* stride = loop_iv_stride(incr, loop, xphi); + Node* stride = loop_iv_stride(incr, xphi); if (stride == nullptr) { return false; @@ -1664,7 +1663,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ jlong stride_con = stride->get_integer_as_long(iv_bt); assert(stride_con != 0, "missed some peephole opt"); - PhiNode* phi = loop_iv_phi(xphi, phi_incr, x, loop); + PhiNode* phi = loop_iv_phi(xphi, phi_incr, x); if (phi == nullptr || (trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != incr) || @@ -2255,7 +2254,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ is_deleteable_safept(sfpt); IdealLoopTree* outer_ilt = nullptr; if (strip_mine_loop) { - outer_ilt = create_outer_strip_mined_loop(test, cmp, init_control, loop, + outer_ilt = create_outer_strip_mined_loop(init_control, loop, cl_prob, le->_fcnt, entry_control, iffalse); } @@ -3949,7 +3948,7 @@ void IdealLoopTree::check_safepts(VectorSet &visited, Node_List &stack) { //---------------------------is_deleteable_safept---------------------------- // Is safept not required by an outer loop? -bool PhaseIdealLoop::is_deleteable_safept(Node* sfpt) { +bool PhaseIdealLoop::is_deleteable_safept(Node* sfpt) const { assert(sfpt->Opcode() == Op_SafePoint, ""); IdealLoopTree* lp = get_loop(sfpt)->_parent; while (lp != nullptr) { diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index f28ad41aad5e6..5fa908c3f4c39 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1000,7 +1000,7 @@ class PhaseIdealLoop : public PhaseTransform { void rewire_old_target_loop_entry_dependency_to_new_entry(LoopNode* target_loop_head, const Node* old_target_loop_entry, uint node_index_before_new_assertion_predicate_nodes); - void insert_loop_limit_check_predicate(ParsePredicateSuccessProj* loop_limit_check_parse_proj, Node* cmp_limit, + void insert_loop_limit_check_predicate(const ParsePredicateSuccessProj* loop_limit_check_parse_proj, Node* cmp_limit, Node* bol); void log_loop_tree(); @@ -1214,7 +1214,7 @@ class PhaseIdealLoop : public PhaseTransform { void recompute_dom_depth(); // Is safept not required by an outer loop? - bool is_deleteable_safept(Node* sfpt); + bool is_deleteable_safept(Node* sfpt) const; // Replace parallel induction variable (parallel to trip counter) void replace_parallel_iv(IdealLoopTree *loop); @@ -1265,23 +1265,23 @@ class PhaseIdealLoop : public PhaseTransform { // Per-Node transform virtual Node* transform(Node* n) { return nullptr; } - Node* loop_exit_control(Node* x, IdealLoopTree* loop); - Node* loop_exit_test(Node* back_control, IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob); - Node* loop_iv_incr(Node* incr, Node* x, IdealLoopTree* loop, Node*& phi_incr); - Node* loop_iv_stride(Node* incr, IdealLoopTree* loop, Node*& xphi); - PhiNode* loop_iv_phi(Node* xphi, Node* phi_incr, Node* x, IdealLoopTree* loop); + Node* loop_exit_control(const Node* x, const IdealLoopTree* loop) const; + Node* loop_exit_test(const Node* back_control, const IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob); + Node* loop_iv_incr(Node* incr, const Node* x, const IdealLoopTree* loop, const Node*& phi_incr); + static Node* loop_iv_stride(const Node* incr, Node*& xphi); + static PhiNode* loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x); bool is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_bt); Node* loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv, Node* outer_phi, Node* inner_head, BasicType bt); bool create_loop_nest(IdealLoopTree* loop, Node_List &old_new); #ifdef ASSERT - bool convert_to_long_loop(Node* cmp, Node* phi, IdealLoopTree* loop); + bool convert_to_long_loop(Node* cmp, const Node* phi, const IdealLoopTree* loop); #endif void add_parse_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop, SafePointNode* sfpt); - SafePointNode* find_safepoint(Node* back_control, Node* x, IdealLoopTree* loop); + SafePointNode* find_safepoint(Node* back_control, const Node* x, const IdealLoopTree* loop); IdealLoopTree* insert_outer_loop(IdealLoopTree* loop, LoopNode* outer_l, Node* outer_ift); - IdealLoopTree* create_outer_strip_mined_loop(BoolNode *test, Node *cmp, Node *init_control, + IdealLoopTree* create_outer_strip_mined_loop(Node *init_control, IdealLoopTree* loop, float cl_prob, float le_fcnt, Node*& entry_control, Node*& iffalse); @@ -1836,6 +1836,9 @@ class PhaseIdealLoop : public PhaseTransform { ConNode* zerocon(BasicType bt); }; +class CountedLoopConverter { + +}; class AutoNodeBudget : public StackObj { From 9b7b7aa829976d2999bdeca069b1b483ea968117 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Wed, 12 Mar 2025 15:50:57 -0400 Subject: [PATCH 02/15] update a caller --- src/hotspot/share/opto/loopnode.cpp | 2 +- src/hotspot/share/opto/loopnode.hpp | 2 +- src/hotspot/share/opto/loopopts.cpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 02c3d4ea19ab6..28b4b59e8af09 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -1611,7 +1611,7 @@ bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, const Node* phi, const Idea #endif //------------------------------is_counted_loop-------------------------------- -bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_bt) { +bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicType iv_bt) { PhaseGVN *gvn = &_igvn; Node* back_control = loop_exit_control(x, loop); diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 5fa908c3f4c39..d4ce0dfeae4cd 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1271,7 +1271,7 @@ class PhaseIdealLoop : public PhaseTransform { static Node* loop_iv_stride(const Node* incr, Node*& xphi); static PhiNode* loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x); - bool is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_bt); + bool is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicType iv_bt); Node* loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv, Node* outer_phi, Node* inner_head, BasicType bt); bool create_loop_nest(IdealLoopTree* loop, Node_List &old_new); diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index 36545870ad86e..68796a0769280 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -4301,13 +4301,13 @@ bool PhaseIdealLoop::duplicate_loop_backedge(IdealLoopTree *loop, Node_List &old } assert(in->Opcode() == Op_AddI, "wrong increment code"); Node* xphi = nullptr; - Node* stride = loop_iv_stride(in, loop, xphi); + Node* stride = loop_iv_stride(in, xphi); if (stride == nullptr) { continue; } - PhiNode* phi = loop_iv_phi(xphi, nullptr, head, loop); + PhiNode* phi = loop_iv_phi(xphi, nullptr, head); if (phi == nullptr || (trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != incr) || (trunc1 != nullptr && phi->in(LoopNode::LoopBackControl) != trunc1)) { From e8016cec8e1781c5246e9f1d0ec9d04447fd1a01 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Mon, 24 Mar 2025 13:50:15 -0400 Subject: [PATCH 03/15] wip --- src/hotspot/share/opto/loopnode.cpp | 242 +++++++++++++++------------- src/hotspot/share/opto/loopnode.hpp | 31 +++- src/hotspot/share/opto/loopopts.cpp | 68 ++++---- 3 files changed, 188 insertions(+), 153 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 28b4b59e8af09..941ebcc7a1e60 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -404,73 +404,71 @@ Node* PhaseIdealLoop::loop_exit_control(const Node* x, const IdealLoopTree* loop return iftrue; } -Node* PhaseIdealLoop::loop_exit_test(const Node* back_control, const IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob) { +//Node* PhaseIdealLoop::loop_exit_test(const Node* back_control, const IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob) { +PhaseIdealLoop::LoopExitTest PhaseIdealLoop::loop_exit_test(const Node* back_control, const IdealLoopTree* loop) { + + const Node* iftrue = back_control; uint iftrue_op = iftrue->Opcode(); Node* iff = iftrue->in(0); BoolNode* test = iff->in(1)->as_Bool(); - bt = test->_test._test; - cl_prob = iff->as_If()->_prob; + BoolTest::mask bt = test->_test._test; + float cl_prob = iff->as_If()->_prob; if (iftrue_op == Op_IfFalse) { bt = BoolTest(bt).negate(); - cl_prob = 1.0 - cl_prob; + cl_prob = 1.0f - cl_prob; } // Get backedge compare Node* cmp = test->in(1); if (!cmp->is_Cmp()) { - return nullptr; + return {}; } // Find the trip-counter increment & limit. Limit must be loop invariant. - incr = cmp->in(1); - limit = cmp->in(2); + Node* incr = cmp->in(1); + Node* limit = cmp->in(2); // --------- // need 'loop()' test to tell if limit is loop invariant // --------- if (!is_member(loop, get_ctrl(incr))) { // Swapped trip counter and limit? - Node* tmp = incr; // Then reverse order into the CmpI - incr = limit; - limit = tmp; + swap(incr, limit); // Then reverse order into the CmpI bt = BoolTest(bt).commute(); // And commute the exit test } if (is_member(loop, get_ctrl(limit))) { // Limit must be loop-invariant - return nullptr; + return {}; } if (!is_member(loop, get_ctrl(incr))) { // Trip counter must be loop-variant - return nullptr; + return {}; } - return cmp; + return {cmp, incr, limit, bt, cl_prob}; } -Node* PhaseIdealLoop::loop_iv_incr(Node* incr, const Node* x, const IdealLoopTree* loop, const Node*& phi_incr) { - if (incr->is_Phi()) { - if (incr->as_Phi()->region() != x || incr->req() != 3) { - return nullptr; // Not simple trip counter expression - } - phi_incr = incr; - incr = phi_incr->in(LoopNode::LoopBackControl); // Assume incr is on backedge of Phi - if (!is_member(loop, get_ctrl(incr))) { // Trip counter must be loop-variant - return nullptr; +PhaseIdealLoop::LoopIVIncr PhaseIdealLoop::loop_iv_incr(Node* incr, const Node* x, const IdealLoopTree* loop) { + if (incr->is_Phi() && incr->as_Phi()->region() == x && incr->req() == 3) { // Simple trip counter expression + Node* phi_incr = incr; + Node* back_control = phi_incr->in(LoopNode::LoopBackControl); // Assume old_incr is on backedge of Phi + if (loop->_phase->is_member(loop, loop->_phase->get_ctrl(back_control))) { // Trip counter must be loop-variant + return {back_control, phi_incr}; } } - return incr; + return {incr, nullptr}; } -Node* PhaseIdealLoop::loop_iv_stride(const Node* incr, Node*& xphi) { +PhaseIdealLoop::LoopIvStride PhaseIdealLoop::loop_iv_stride(const Node* incr) { assert(incr->Opcode() == Op_AddI || incr->Opcode() == Op_AddL, "caller resp."); // Get merge point - xphi = incr->in(1); + Node* xphi = incr->in(1); Node *stride = incr->in(2); if (!stride->is_Con()) { // Oops, swap these if (!xphi->is_Con()) { // Is the other guy a constant? - return nullptr; // Nope, unknown stride, bail out + return {}; // Nope, unknown stride, bail out } swap(xphi, stride); // 'incr' is commutative, so ok to swap } - return stride; + return {stride, xphi}; } PhiNode* PhaseIdealLoop::loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x) { @@ -1467,34 +1465,39 @@ void PhaseIdealLoop::check_counted_loop_shape(IdealLoopTree* loop, Node* x, Basi Node* back_control = loop_exit_control(x, loop); assert(back_control != nullptr, "no back control"); - BoolTest::mask mask = BoolTest::illegal; - float cl_prob = 0; - Node* incr = nullptr; - Node* limit = nullptr; +// BoolTest::mask mask = BoolTest::illegal; +// float cl_prob = 0; +// Node* incr = nullptr; +// Node* limit = nullptr; - Node* cmp = loop_exit_test(back_control, loop, incr, limit, mask, cl_prob); - assert(cmp != nullptr && cmp->Opcode() == Op_Cmp(bt), "no exit test"); +// Node* cmp = loop_exit_test(back_control, loop, incr, limit, mask, cl_prob); + LoopExitTest exit_test = loop_exit_test(back_control, loop); + assert(exit_test.cmp != nullptr && exit_test.cmp->Opcode() == Op_Cmp(bt), "no exit test"); - const Node* phi_incr = nullptr; - incr = loop_iv_incr(incr, x, loop, phi_incr); - assert(incr != nullptr && incr->Opcode() == Op_Add(bt), "no incr"); +// const Node* phi_incr = incr; +// incr = loop_iv_incr(incr, x, loop); + const LoopIVIncr iv_incr = loop_iv_incr(exit_test.incr, x, loop); +// phi_incr = iv_incr.phi_incr; +// incr = iv_incr.incr; + assert(iv_incr.incr != nullptr && iv_incr.incr->Opcode() == Op_Add(bt), "no incr"); - Node* xphi = nullptr; - Node* stride = loop_iv_stride(incr, xphi); +// Node* xphi = nullptr; +// Node* stride = loop_iv_stride(iv_incr.incr, xphi); + LoopIvStride stride = loop_iv_stride(iv_incr.incr); - assert(stride != nullptr, "no stride"); + assert(stride.stride != nullptr, "no stride"); - PhiNode* phi = loop_iv_phi(xphi, phi_incr, x); + PhiNode* phi = loop_iv_phi(stride.xphi, iv_incr.phi_incr, x); - assert(phi != nullptr && phi->in(LoopNode::LoopBackControl) == incr, "No phi"); + assert(phi != nullptr && phi->in(LoopNode::LoopBackControl) == iv_incr.incr, "No phi"); - jlong stride_con = stride->get_integer_as_long(bt); + jlong stride_con = stride.stride->get_integer_as_long(bt); - assert(condition_stride_ok(mask, stride_con), "illegal condition"); + assert(condition_stride_ok(exit_test.mask, stride_con), "illegal condition"); - assert(mask != BoolTest::ne, "unexpected condition"); - assert(phi_incr == nullptr, "bad loop shape"); - assert(cmp->in(1) == incr, "bad exit test shape"); + assert(exit_test.mask != BoolTest::ne, "unexpected condition"); + assert(iv_incr.phi_incr == nullptr, "bad loop shape"); + assert(exit_test.cmp->in(1) == iv_incr.incr, "bad exit test shape"); // Safepoint on backedge not supported assert(x->in(LoopNode::LoopBackControl)->Opcode() != Op_SafePoint, "no safepoint on backedge"); @@ -1619,55 +1622,64 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy return false; } - BoolTest::mask bt = BoolTest::illegal; - float cl_prob = 0; - Node* incr = nullptr; - Node* limit = nullptr; - Node* cmp = loop_exit_test(back_control, loop, incr, limit, bt, cl_prob); - if (cmp == nullptr || cmp->Opcode() != Op_Cmp(iv_bt)) { +// BoolTest::mask bt = BoolTest::illegal; +// float cl_prob = 0; +// Node* incr = nullptr; +// Node* limit = nullptr; +// Node* cmp = loop_exit_test(back_control, loop, incr, limit, bt, cl_prob); + LoopExitTest exit_test = loop_exit_test(back_control, loop); + if (exit_test.cmp == nullptr || exit_test.cmp->Opcode() != Op_Cmp(iv_bt)) { return false; // Avoid pointer & float & 64-bit compares } // Trip-counter increment must be commutative & associative. - if (incr->Opcode() == Op_Cast(iv_bt)) { + Node* incr = exit_test.incr; + if (exit_test.incr->Opcode() == Op_Cast(iv_bt)) { incr = incr->in(1); } - const Node* phi_incr = nullptr; - incr = loop_iv_incr(incr, x, loop, phi_incr); - if (incr == nullptr) { +// const Node* phi_incr = incr; +// incr = loop_iv_incr(incr, x, loop); + const LoopIVIncr iv_incr = loop_iv_incr(incr, x, loop); +// phi_incr = iv_incr.phi_incr; +// incr = iv_incr.incr; + if (iv_incr.incr == nullptr) { return false; } - Node* trunc1 = nullptr; - Node* trunc2 = nullptr; - const TypeInteger* iv_trunc_t = nullptr; - Node* orig_incr = incr; - if (!(incr = CountedLoopNode::match_incr_with_optional_truncation(incr, &trunc1, &trunc2, &iv_trunc_t, iv_bt))) { +// Node* trunc1 = nullptr; +// Node* trunc2 = nullptr; +// const TypeInteger* iv_trunc_t = nullptr; +// Node* orig_incr = iv_incr.incr; // FIXME: useless alias? +// if (!(incr = CountedLoopNode::match_incr_with_optional_truncation(incr, &trunc1, &trunc2, &iv_trunc_t, iv_bt))) { + CountedLoopNode::TruncatedIncrement increment = CountedLoopNode::match_incr_with_optional_truncation(iv_incr.incr, iv_bt); + if (increment.incr == nullptr) { return false; // Funny increment opcode } - assert(incr->Opcode() == Op_Add(iv_bt), "wrong increment code"); + assert(increment.incr->Opcode() == Op_Add(iv_bt), "wrong increment code"); - Node* xphi = nullptr; - Node* stride = loop_iv_stride(incr, xphi); +// Node* xphi = nullptr; +// Node* stride = loop_iv_stride(increment.incr, xphi); + const LoopIvStride stride = loop_iv_stride(increment.incr); - if (stride == nullptr) { + if (stride.stride == nullptr) { return false; } + Node* xphi = stride.xphi; if (xphi->Opcode() == Op_Cast(iv_bt)) { xphi = xphi->in(1); } // Stride must be constant - jlong stride_con = stride->get_integer_as_long(iv_bt); + jlong stride_con = stride.stride->get_integer_as_long(iv_bt); assert(stride_con != 0, "missed some peephole opt"); - PhiNode* phi = loop_iv_phi(xphi, phi_incr, x); + PhiNode* phi = loop_iv_phi(xphi, iv_incr.phi_incr, x); if (phi == nullptr || - (trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != incr) || - (trunc1 != nullptr && phi->in(LoopNode::LoopBackControl) != trunc1)) { + (increment.trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != increment.incr) || + (increment.trunc1 != nullptr && phi->in(LoopNode::LoopBackControl) != increment.trunc1)) { return false; } @@ -1676,8 +1688,8 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy Node* iff = iftrue->in(0); BoolNode* test = iff->in(1)->as_Bool(); - const TypeInteger* limit_t = gvn->type(limit)->is_integer(iv_bt); - if (trunc1 != nullptr) { + const TypeInteger* limit_t = gvn->type(exit_test.limit)->is_integer(iv_bt); + if (increment.trunc1 != nullptr) { // When there is a truncation, we must be sure that after the truncation // the trip counter will end up higher than the limit, otherwise we are looking // at an endless loop. Can happen with range checks. @@ -1695,9 +1707,10 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy // If the array is longer then this is an endless loop // - No transformation can be done. - const TypeInteger* incr_t = gvn->type(orig_incr)->is_integer(iv_bt); +// const TypeInteger* incr_t = gvn->type(orig_incr)->is_integer(iv_bt); + const TypeInteger* incr_t = gvn->type(iv_incr.incr)->is_integer(iv_bt); if (limit_t->hi_as_long() > incr_t->hi_as_long()) { - // if the limit can have a higher value than the increment (before the phi) + // if the limit can have a higher value than the increment (before the0 phi) return false; } } @@ -1705,8 +1718,8 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy Node *init_trip = phi->in(LoopNode::EntryControl); // If iv trunc type is smaller than int, check for possible wrap. - if (!TypeInteger::bottom(iv_bt)->higher_equal(iv_trunc_t)) { - assert(trunc1 != nullptr, "must have found some truncation"); + if (!TypeInteger::bottom(iv_bt)->higher_equal(increment.trunc_type)) { + assert(increment.trunc1 != nullptr, "must have found some truncation"); // Get a better type for the phi (filtered thru if's) const TypeInteger* phi_ft = filtered_type(phi); @@ -1723,23 +1736,23 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy // ensure no truncation occurs after the increment. if (stride_con > 0) { - if (iv_trunc_t->hi_as_long() - phi_ft->hi_as_long() < stride_con || - iv_trunc_t->lo_as_long() > phi_ft->lo_as_long()) { + if (increment.trunc_type->hi_as_long() - phi_ft->hi_as_long() < stride_con || + increment.trunc_type->lo_as_long() > phi_ft->lo_as_long()) { return false; // truncation may occur } } else if (stride_con < 0) { - if (iv_trunc_t->lo_as_long() - phi_ft->lo_as_long() > stride_con || - iv_trunc_t->hi_as_long() < phi_ft->hi_as_long()) { + if (increment.trunc_type->lo_as_long() - phi_ft->lo_as_long() > stride_con || + increment.trunc_type->hi_as_long() < phi_ft->hi_as_long()) { return false; // truncation may occur } } // No possibility of wrap so truncation can be discarded // Promote iv type to Int } else { - assert(trunc1 == nullptr && trunc2 == nullptr, "no truncation for int"); + assert(increment.trunc1 == nullptr && increment.trunc2 == nullptr, "no truncation for int"); } - if (!condition_stride_ok(bt, stride_con)) { + if (!condition_stride_ok(exit_test.mask, stride_con)) { return false; } @@ -1755,7 +1768,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy } } - if (phi_incr != nullptr && bt != BoolTest::ne) { + if (iv_incr.phi_incr != nullptr && exit_test.mask != BoolTest::ne) { // check if there is a possibility of IV overflowing after the first increment if (stride_con > 0) { if (init_t->hi_as_long() > max_signed_integer(iv_bt) - stride_con) { @@ -1982,10 +1995,10 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy } // Accounting for (LE3) and (LE4) where we use pre-incremented phis in the loop exit check. - const jlong limit_correction_for_pre_iv_exit_check = (phi_incr != nullptr) ? stride_con : 0; + const jlong limit_correction_for_pre_iv_exit_check = (iv_incr.phi_incr != nullptr) ? stride_con : 0; // Accounting for (LE2) and (LE4) where we use <= or >= in the loop exit check. - const bool includes_limit = (bt == BoolTest::le || bt == BoolTest::ge); + const bool includes_limit = (exit_test.mask == BoolTest::le || exit_test.mask == BoolTest::ge); const jlong limit_correction_for_le_ge_exit_check = (includes_limit ? (stride_con > 0 ? 1 : -1) : 0); const jlong limit_correction = limit_correction_for_pre_iv_exit_check + limit_correction_for_le_ge_exit_check; @@ -2019,7 +2032,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy } ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); - if (!is_dominator(get_ctrl(limit), loop_limit_check_parse_predicate->in(0))) { + if (!is_dominator(get_ctrl(exit_test.limit), loop_limit_check_parse_predicate->in(0))) { return false; } @@ -2027,10 +2040,10 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy Node* bol; if (stride_con > 0) { - cmp_limit = CmpNode::make(limit, _igvn.integercon(max_signed_integer(iv_bt) - final_correction, iv_bt), iv_bt); + cmp_limit = CmpNode::make(exit_test.limit, _igvn.integercon(max_signed_integer(iv_bt) - final_correction, iv_bt), iv_bt); bol = new BoolNode(cmp_limit, BoolTest::le); } else { - cmp_limit = CmpNode::make(limit, _igvn.integercon(min_signed_integer(iv_bt) - final_correction, iv_bt), iv_bt); + cmp_limit = CmpNode::make(exit_test.limit, _igvn.integercon(min_signed_integer(iv_bt) - final_correction, iv_bt), iv_bt); bol = new BoolNode(cmp_limit, BoolTest::ge); } @@ -2046,8 +2059,8 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy (stride_con < 0 && init_t->lo_as_long() <= limit_t->hi_as_long()); if (init_gte_limit && // (2.1) - ((bt == BoolTest::ne || init_plus_stride_could_overflow) && // (2.3) - !has_dominating_loop_limit_check(init_trip, limit, stride_con, iv_bt, init_control))) { // (2.2) + ((exit_test.mask == BoolTest::ne || init_plus_stride_could_overflow) && // (2.3) + !has_dominating_loop_limit_check(init_trip, exit_test.limit, stride_con, iv_bt, init_control))) { // (2.2) // (2) Iteration Loop Limit Check Predicate is required because neither (2.1), (2.2), nor (2.3) holds. // We use the following condition: // - stride > 0: init < limit @@ -2073,7 +2086,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); Node* parse_predicate_entry = loop_limit_check_parse_predicate->in(0); - if (!is_dominator(get_ctrl(limit), parse_predicate_entry) || + if (!is_dominator(get_ctrl(exit_test.limit), parse_predicate_entry) || !is_dominator(get_ctrl(init_trip), parse_predicate_entry)) { return false; } @@ -2082,16 +2095,17 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy Node* bol; if (stride_con > 0) { - cmp_limit = CmpNode::make(init_trip, limit, iv_bt); + cmp_limit = CmpNode::make(init_trip, exit_test.limit, iv_bt); bol = new BoolNode(cmp_limit, BoolTest::lt); } else { - cmp_limit = CmpNode::make(init_trip, limit, iv_bt); + cmp_limit = CmpNode::make(init_trip, exit_test.limit, iv_bt); bol = new BoolNode(cmp_limit, BoolTest::gt); } insert_loop_limit_check_predicate(init_control->as_IfTrue(), cmp_limit, bol); } + BoolTest::mask bt = exit_test.mask; // TODO: need to save if (bt == BoolTest::ne) { // Now we need to canonicalize the loop condition if it is 'ne'. assert(stride_con == 1 || stride_con == -1, "simple increment only - checked before"); @@ -2140,14 +2154,14 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy if (iv_bt == T_INT && !x->as_Loop()->is_loop_nest_inner_loop() && StressLongCountedLoop > 0 && - trunc1 == nullptr && - convert_to_long_loop(cmp, phi, loop)) { + increment.trunc1 == nullptr && + convert_to_long_loop(exit_test.cmp, phi, loop)) { return false; } #endif - Node* adjusted_limit = limit; - if (phi_incr != nullptr) { + Node* adjusted_limit = exit_test.limit; // TODO: need to save + if (iv_incr.phi_incr != nullptr) { // If compare points directly to the phi we need to adjust // the compare so that it points to the incr. Limit have // to be adjusted to keep trip count the same and we @@ -2157,7 +2171,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy // is converted to // i = init; do {} while(++i < limit+1); // - adjusted_limit = gvn->transform(AddNode::make(limit, stride, iv_bt)); + adjusted_limit = gvn->transform(AddNode::make(exit_test.limit, stride.stride, iv_bt)); } if (includes_limit) { @@ -2177,13 +2191,14 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy // Build a canonical trip test. // Clone code, as old values may be in use. - incr = incr->clone(); - incr->set_req(1,phi); - incr->set_req(2,stride); - incr = _igvn.register_new_node_with_optimizer(incr); - set_early_ctrl(incr, false); +// incr = incr->clone(); + Node* incr_clone = increment.incr->clone(); + incr_clone->set_req(1,phi); + incr_clone->set_req(2, stride.stride); + incr_clone = _igvn.register_new_node_with_optimizer(incr_clone); + set_early_ctrl(incr_clone, false); _igvn.rehash_node_delayed(phi); - phi->set_req_X( LoopNode::LoopBackControl, incr, &_igvn ); + phi->set_req_X( LoopNode::LoopBackControl, incr_clone, &_igvn ); // If phi type is more restrictive than Int, raise to // Int to prevent (almost) infinite recursion in igvn @@ -2196,8 +2211,8 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy _igvn.replace_node(phi, nphi); phi = nphi->as_Phi(); } - cmp = cmp->clone(); - cmp->set_req(1,incr); + Node* cmp = exit_test.cmp->clone(); // TODO: need to save + cmp->set_req(1,incr_clone); cmp->set_req(2, adjusted_limit); cmp = _igvn.register_new_node_with_optimizer(cmp); set_ctrl(cmp, iff->in(0)); @@ -2209,7 +2224,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy set_ctrl(test, iff->in(0)); // Replace the old IfNode with a new LoopEndNode - Node *lex = _igvn.register_new_node_with_optimizer(BaseCountedLoopEndNode::make(iff->in(0), test, cl_prob, iff->as_If()->_fcnt, iv_bt)); + Node *lex = _igvn.register_new_node_with_optimizer(BaseCountedLoopEndNode::make(iff->in(0), test, exit_test.cl_prob, iff->as_If()->_fcnt, iv_bt)); IfNode *le = lex->as_If(); uint dd = dom_depth(iff); set_idom(le, le->in(0), dd); // Update dominance for loop exit @@ -2255,7 +2270,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy IdealLoopTree* outer_ilt = nullptr; if (strip_mine_loop) { outer_ilt = create_outer_strip_mined_loop(init_control, loop, - cl_prob, le->_fcnt, entry_control, + exit_test.cl_prob, le->_fcnt, entry_control, iffalse); } @@ -2676,11 +2691,9 @@ Node* LoopLimitNode::Identity(PhaseGVN* phase) { // Match increment with optional truncation: // CHAR: (i+1)&0x7fff, BYTE: ((i+1)<<8)>>8, or SHORT: ((i+1)<<16)>>16 // Return null for failure. Success returns the increment node. -Node* CountedLoopNode::match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, - const TypeInteger** trunc_type, - BasicType bt) { +CountedLoopNode::TruncatedIncrement CountedLoopNode::match_incr_with_optional_truncation(Node* expr, BasicType bt) { // Quick cutouts: - if (expr == nullptr || expr->req() != 3) return nullptr; + if (expr == nullptr || expr->req() != 3) return TruncatedIncrement{}; Node *t1 = nullptr; Node *t2 = nullptr; @@ -2721,14 +2734,11 @@ Node* CountedLoopNode::match_incr_with_optional_truncation(Node* expr, Node** tr // If (maybe after stripping) it is an AddI, we won: if (n1op == Op_Add(bt)) { - *trunc1 = t1; - *trunc2 = t2; - *trunc_type = trunc_t; - return n1; + return TruncatedIncrement{n1, t1, t2, trunc_t}; } // failed - return nullptr; + return TruncatedIncrement{}; } IfNode* CountedLoopNode::find_multiversion_if_from_multiversion_fast_main_loop() { diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index d4ce0dfeae4cd..c4f9d1ba659a0 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -258,9 +258,13 @@ class CountedLoopNode : public BaseCountedLoopNode { int stride_con() const; // Match increment with optional truncation - static Node* - match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, const TypeInteger** trunc_type, - BasicType bt); + struct TruncatedIncrement { + Node* incr = nullptr; + Node* trunc1 = nullptr; + Node* trunc2 = nullptr; + const TypeInteger* trunc_type = nullptr; + }; + static TruncatedIncrement match_incr_with_optional_truncation(Node* expr, BasicType bt); // A 'main' loop has a pre-loop and a post-loop. The 'main' loop // can run short a few iterations and may start a few iterations in. @@ -1266,9 +1270,24 @@ class PhaseIdealLoop : public PhaseTransform { virtual Node* transform(Node* n) { return nullptr; } Node* loop_exit_control(const Node* x, const IdealLoopTree* loop) const; - Node* loop_exit_test(const Node* back_control, const IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob); - Node* loop_iv_incr(Node* incr, const Node* x, const IdealLoopTree* loop, const Node*& phi_incr); - static Node* loop_iv_stride(const Node* incr, Node*& xphi); + struct LoopExitTest { + Node* cmp = nullptr; + Node* incr = nullptr; + Node* limit = nullptr; + const BoolTest::mask mask = BoolTest::illegal; + const float cl_prob = 0.0f; + }; + LoopExitTest loop_exit_test(const Node* back_control, const IdealLoopTree* loop); + struct LoopIVIncr { + Node* incr = nullptr; + Node* phi_incr = nullptr; + }; + LoopIVIncr loop_iv_incr(Node* old_incr, const Node* x, const IdealLoopTree* loop); + struct LoopIvStride { + Node* stride = nullptr; + Node* xphi = nullptr; + }; + static LoopIvStride loop_iv_stride(const Node* incr); static PhiNode* loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x); bool is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicType iv_bt); diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index 68796a0769280..c1f44c5631cc6 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -2848,23 +2848,25 @@ int PhaseIdealLoop::stride_of_possible_iv(Node* iff) { Node* phi = cmp1; for (uint i = 1; i < phi->req(); i++) { Node* in = phi->in(i); - Node* add = CountedLoopNode::match_incr_with_optional_truncation(in, - &trunc1, &trunc2, &ttype, T_INT); - if (add && add->in(1) == phi) { - add2 = add->in(2); +// Node* add = CountedLoopNode::match_incr_with_optional_truncation(in, +// &trunc1, &trunc2, &ttype, T_INT); + CountedLoopNode::TruncatedIncrement increment = CountedLoopNode::match_incr_with_optional_truncation(in, T_INT); + if (increment.incr != nullptr && increment.incr->in(1) == phi) { + add2 = increment.incr->in(2); break; } } } else { // (If (Bool (CmpX addtrunc:(Optional-trunc((AddI (Phi ...addtrunc...) add2)) ))) Node* addtrunc = cmp1; - Node* add = CountedLoopNode::match_incr_with_optional_truncation(addtrunc, - &trunc1, &trunc2, &ttype, T_INT); - if (add && add->in(1)->is_Phi()) { - Node* phi = add->in(1); +// Node* add = CountedLoopNode::match_incr_with_optional_truncation(addtrunc, +// &trunc1, &trunc2, &ttype, T_INT); + CountedLoopNode::TruncatedIncrement increment = CountedLoopNode::match_incr_with_optional_truncation(addtrunc, T_INT); + if (increment.incr != nullptr && increment.incr->in(1)->is_Phi()) { + Node* phi = increment.incr->in(1); for (uint i = 1; i < phi->req(); i++) { if (phi->in(i) == addtrunc) { - add2 = add->in(2); + add2 = increment.incr->in(2); break; } } @@ -4268,49 +4270,53 @@ bool PhaseIdealLoop::duplicate_loop_backedge(IdealLoopTree *loop, Node_List &old return false; } - BoolTest::mask bt = BoolTest::illegal; - float cl_prob = 0; - Node* incr = nullptr; - Node* limit = nullptr; - Node* cmp = loop_exit_test(back_control, loop, incr, limit, bt, cl_prob); - if (cmp == nullptr || cmp->Opcode() != Op_CmpI) { +// BoolTest::mask bt = BoolTest::illegal; +// float cl_prob = 0; +// Node* incr = nullptr; +// Node* limit = nullptr; +// Node* cmp = loop_exit_test(back_control, loop, incr, limit, bt, cl_prob); + LoopExitTest loop_exit = loop_exit_test(back_control, loop); + if (loop_exit.cmp == nullptr || loop_exit.cmp->Opcode() != Op_CmpI) { return false; } // With an extra phi for the candidate iv? // Or the region node is the loop head - if (!incr->is_Phi() || incr->in(0) == head) { + if (!loop_exit.incr->is_Phi() || loop_exit.incr->in(0) == head) { return false; } PathFrequency pf(head, this); - region = incr->in(0); + region = loop_exit.incr->in(0); // Go over all paths for the extra phi's region and see if that // path is frequent enough and would match the expected iv shape // if the extra phi is removed inner = 0; - for (uint i = 1; i < incr->req(); ++i) { - Node* in = incr->in(i); - Node* trunc1 = nullptr; - Node* trunc2 = nullptr; - const TypeInteger* iv_trunc_t = nullptr; - Node* orig_in = in; - if (!(in = CountedLoopNode::match_incr_with_optional_truncation(in, &trunc1, &trunc2, &iv_trunc_t, T_INT))) { + for (uint i = 1; i < loop_exit.incr->req(); ++i) { +// Node* in = incr->in(i); +// Node* trunc1 = nullptr; +// Node* trunc2 = nullptr; +// const TypeInteger* iv_trunc_t = nullptr; +// Node* orig_in = in; + CountedLoopNode::TruncatedIncrement increment = CountedLoopNode::match_incr_with_optional_truncation(loop_exit.incr->in(i), T_INT); +// if (!(in = CountedLoopNode::match_incr_with_optional_truncation(in, &trunc1, &trunc2, &iv_trunc_t, T_INT))) { + if (increment.incr == nullptr) { continue; } - assert(in->Opcode() == Op_AddI, "wrong increment code"); - Node* xphi = nullptr; - Node* stride = loop_iv_stride(in, xphi); + assert(increment.incr->Opcode() == Op_AddI, "wrong increment code"); +// Node* xphi = nullptr; +// Node* stride = loop_iv_stride(increment.incr, xphi); + LoopIvStride stride = loop_iv_stride(increment.incr); - if (stride == nullptr) { + if (stride.stride == nullptr) { continue; } - PhiNode* phi = loop_iv_phi(xphi, nullptr, head); + PhiNode* phi = loop_iv_phi(stride.xphi, nullptr, head); if (phi == nullptr || - (trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != incr) || - (trunc1 != nullptr && phi->in(LoopNode::LoopBackControl) != trunc1)) { + (increment.trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != loop_exit.incr) || + (increment.trunc1 != nullptr && phi->in(LoopNode::LoopBackControl) != increment.trunc1)) { return false; } From e73066589edc18aa391c7ba8b5bd1484ff6d3ac5 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Mon, 31 Mar 2025 15:11:55 -0400 Subject: [PATCH 04/15] wip: create CountedLoopConverter --- src/hotspot/share/opto/loopnode.cpp | 937 +++++++++++++++++++++++++--- src/hotspot/share/opto/loopnode.hpp | 49 +- 2 files changed, 897 insertions(+), 89 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 941ebcc7a1e60..a5adc1959e9a7 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -1614,19 +1614,24 @@ bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, const Node* phi, const Idea #endif //------------------------------is_counted_loop-------------------------------- -bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicType iv_bt) { +bool PhaseIdealLoop::is_counted_loop(Node* head, IdealLoopTree*& loop, const BasicType iv_bt) { + if (UseNewCode || true) { // FIXME: remove + CountedLoopConverter converter(this, head, loop, iv_bt); + if (converter.is_counted_loop()) { + converter.convert(); + return true; + } + + return false; + } + PhaseGVN *gvn = &_igvn; - Node* back_control = loop_exit_control(x, loop); + Node* back_control = loop_exit_control(head, loop); if (back_control == nullptr) { return false; } -// BoolTest::mask bt = BoolTest::illegal; -// float cl_prob = 0; -// Node* incr = nullptr; -// Node* limit = nullptr; -// Node* cmp = loop_exit_test(back_control, loop, incr, limit, bt, cl_prob); LoopExitTest exit_test = loop_exit_test(back_control, loop); if (exit_test.cmp == nullptr || exit_test.cmp->Opcode() != Op_Cmp(iv_bt)) { return false; // Avoid pointer & float & 64-bit compares @@ -1638,28 +1643,17 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy incr = incr->in(1); } -// const Node* phi_incr = incr; -// incr = loop_iv_incr(incr, x, loop); - const LoopIVIncr iv_incr = loop_iv_incr(incr, x, loop); -// phi_incr = iv_incr.phi_incr; -// incr = iv_incr.incr; + const LoopIVIncr iv_incr = loop_iv_incr(incr, head, loop); if (iv_incr.incr == nullptr) { return false; } -// Node* trunc1 = nullptr; -// Node* trunc2 = nullptr; -// const TypeInteger* iv_trunc_t = nullptr; -// Node* orig_incr = iv_incr.incr; // FIXME: useless alias? -// if (!(incr = CountedLoopNode::match_incr_with_optional_truncation(incr, &trunc1, &trunc2, &iv_trunc_t, iv_bt))) { CountedLoopNode::TruncatedIncrement increment = CountedLoopNode::match_incr_with_optional_truncation(iv_incr.incr, iv_bt); if (increment.incr == nullptr) { return false; // Funny increment opcode } assert(increment.incr->Opcode() == Op_Add(iv_bt), "wrong increment code"); -// Node* xphi = nullptr; -// Node* stride = loop_iv_stride(increment.incr, xphi); const LoopIvStride stride = loop_iv_stride(increment.incr); if (stride.stride == nullptr) { @@ -1675,7 +1669,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy jlong stride_con = stride.stride->get_integer_as_long(iv_bt); assert(stride_con != 0, "missed some peephole opt"); - PhiNode* phi = loop_iv_phi(xphi, iv_incr.phi_incr, x); + PhiNode* phi = loop_iv_phi(xphi, iv_incr.phi_incr, head); if (phi == nullptr || (increment.trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != increment.incr) || @@ -1707,7 +1701,6 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy // If the array is longer then this is an endless loop // - No transformation can be done. -// const TypeInteger* incr_t = gvn->type(orig_incr)->is_integer(iv_bt); const TypeInteger* incr_t = gvn->type(iv_incr.incr)->is_integer(iv_bt); if (limit_t->hi_as_long() > incr_t->hi_as_long()) { // if the limit can have a higher value than the increment (before the0 phi) @@ -1785,22 +1778,22 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy // ---- SUCCESS! Found A Trip-Counted Loop! ----- // - if (x->Opcode() == Op_Region) { - // x has not yet been transformed to Loop or LongCountedLoop. + if (head->Opcode() == Op_Region) { + // head has not yet been transformed to Loop or LongCountedLoop. // This should only happen if we are inside an infinite loop. // It happens like this: // build_loop_tree -> do not attach infinite loop and nested loops // beautify_loops -> does not transform the infinite and nested loops to LoopNode, because not attached yet // build_loop_tree -> find and attach infinite and nested loops // counted_loop -> nested Regions are not yet transformed to LoopNodes, we land here - assert(x->as_Region()->is_in_infinite_subgraph(), - "x can only be a Region and not Loop if inside infinite loop"); + assert(head->as_Region()->is_in_infinite_subgraph(), + "head can only be a Region and not Loop if inside infinite loop"); // Come back later when Region is transformed to LoopNode return false; } - assert(x->Opcode() == Op_Loop || x->Opcode() == Op_LongCountedLoop, "regular loops only"); - C->print_method(PHASE_BEFORE_CLOOPS, 3, x); + assert(head->Opcode() == Op_Loop || head->Opcode() == Op_LongCountedLoop, "regular loops only"); + C->print_method(PHASE_BEFORE_CLOOPS, 3, head); // =================================================== // We can only convert this loop to a counted loop if we can guarantee that the iv phi will never overflow at runtime. @@ -1998,7 +1991,8 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy const jlong limit_correction_for_pre_iv_exit_check = (iv_incr.phi_incr != nullptr) ? stride_con : 0; // Accounting for (LE2) and (LE4) where we use <= or >= in the loop exit check. - const bool includes_limit = (exit_test.mask == BoolTest::le || exit_test.mask == BoolTest::ge); + const bool + includes_limit = (exit_test.mask == BoolTest::le || exit_test.mask == BoolTest::ge); // TODO: save to context const jlong limit_correction_for_le_ge_exit_check = (includes_limit ? (stride_con > 0 ? 1 : -1) : 0); const jlong limit_correction = limit_correction_for_pre_iv_exit_check + limit_correction_for_le_ge_exit_check; @@ -2006,26 +2000,32 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy const jlong final_correction = canonicalized_correction + limit_correction; int sov = check_stride_overflow(final_correction, limit_t, iv_bt); - Node* init_control = x->in(LoopNode::EntryControl); + Node* init_control = head->in(LoopNode::EntryControl); + + const Predicates predicates(init_control); + const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); - // If sov==0, limit's type always satisfies the condition, for + // TODO: add to context + bool insert_stride_overflow_limit_check = false; // TODO: replace with sov < 0? + bool insert_init_trip_limit_check = false; + + if (sov < 0) { + return false; // Bailout: integer overflow is certain. + } + + // If sov == 0, limit's type always satisfies the condition, for // example, when it is an array length. - if (sov != 0) { - if (sov < 0) { - return false; // Bailout: integer overflow is certain. - } + if (sov > 0) { // (1) Loop Limit Check Predicate is required because we could not statically prove that // limit + final_correction = adjusted_limit - 1 + stride <= max_int - assert(!x->as_Loop()->is_loop_nest_inner_loop(), "loop was transformed"); - const Predicates predicates(init_control); - const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); + assert(!head->as_Loop()->is_loop_nest_inner_loop(), "loop was transformed"); if (!loop_limit_check_predicate_block->has_parse_predicate()) { // The Loop Limit Check Parse Predicate is not generated if this method trapped here before. #ifdef ASSERT if (TraceLoopLimitCheck) { tty->print("Missing Loop Limit Check Parse Predicate:"); loop->dump_head(); - x->dump(1); + head->dump(1); } #endif return false; @@ -2036,18 +2036,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy return false; } - Node* cmp_limit; - Node* bol; - - if (stride_con > 0) { - cmp_limit = CmpNode::make(exit_test.limit, _igvn.integercon(max_signed_integer(iv_bt) - final_correction, iv_bt), iv_bt); - bol = new BoolNode(cmp_limit, BoolTest::le); - } else { - cmp_limit = CmpNode::make(exit_test.limit, _igvn.integercon(min_signed_integer(iv_bt) - final_correction, iv_bt), iv_bt); - bol = new BoolNode(cmp_limit, BoolTest::ge); - } - - insert_loop_limit_check_predicate(init_control->as_IfTrue(), cmp_limit, bol); + insert_stride_overflow_limit_check = true; } // (2.3) @@ -2070,15 +2059,13 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy // a requirement). We transform the loop exit check by using a less-than-operator. By doing so, we must always // check that init < limit. Otherwise, we could have a different number of iterations at runtime. - const Predicates predicates(init_control); - const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); if (!loop_limit_check_predicate_block->has_parse_predicate()) { // The Loop Limit Check Parse Predicate is not generated if this method trapped here before. #ifdef ASSERT if (TraceLoopLimitCheck) { tty->print("Missing Loop Limit Check Parse Predicate:"); loop->dump_head(); - x->dump(1); + head->dump(1); } #endif return false; @@ -2091,18 +2078,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy return false; } - Node* cmp_limit; - Node* bol; - - if (stride_con > 0) { - cmp_limit = CmpNode::make(init_trip, exit_test.limit, iv_bt); - bol = new BoolNode(cmp_limit, BoolTest::lt); - } else { - cmp_limit = CmpNode::make(init_trip, exit_test.limit, iv_bt); - bol = new BoolNode(cmp_limit, BoolTest::gt); - } - - insert_loop_limit_check_predicate(init_control->as_IfTrue(), cmp_limit, bol); + insert_init_trip_limit_check = true; } BoolTest::mask bt = exit_test.mask; // TODO: need to save @@ -2119,9 +2095,9 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy } } - Node* sfpt = nullptr; + Node* sfpt = nullptr; // TODO: save sfpt if (loop->_child == nullptr) { - sfpt = find_safepoint(back_control, x, loop); + sfpt = find_safepoint(back_control, head, loop); } else { sfpt = iff->in(0); if (sfpt->Opcode() != Op_SafePoint) { @@ -2129,8 +2105,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy } } - if (x->in(LoopNode::LoopBackControl)->Opcode() == Op_SafePoint) { - Node* backedge_sfpt = x->in(LoopNode::LoopBackControl); + if (head->in(LoopNode::LoopBackControl)->Opcode() == Op_SafePoint) { if (((iv_bt == T_INT && LoopStripMiningIter != 0) || iv_bt == T_LONG) && sfpt == nullptr) { @@ -2140,19 +2115,13 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy // location. Give up on that loop. return false; } - if (is_deleteable_safept(backedge_sfpt)) { - lazy_replace(backedge_sfpt, iftrue); - if (loop->_safepts != nullptr) { - loop->_safepts->yank(backedge_sfpt); - } - loop->_tail = iftrue; - } } #ifdef ASSERT + // Stress and convert to long loop if StressLongCountedLoop flag is set if (iv_bt == T_INT && - !x->as_Loop()->is_loop_nest_inner_loop() && + !head->as_Loop()->is_loop_nest_inner_loop() && StressLongCountedLoop > 0 && increment.trunc1 == nullptr && convert_to_long_loop(exit_test.cmp, phi, loop)) { @@ -2160,7 +2129,36 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy } #endif - Node* adjusted_limit = exit_test.limit; // TODO: need to save + // return true; + + // TODO: extract to separate function? + if (insert_stride_overflow_limit_check) { + Node* cmp_limit = CmpNode::make(exit_test.limit, _igvn.integercon((stride_con > 0 + ? max_signed_integer(iv_bt) + : min_signed_integer(iv_bt)) + - final_correction, iv_bt), iv_bt); + Node* bol = new BoolNode(cmp_limit, stride_con > 0 ? BoolTest::le : BoolTest::ge); + insert_loop_limit_check_predicate(init_control->as_IfTrue(), cmp_limit, bol); + } + + if (insert_init_trip_limit_check) { + Node* cmp_limit = CmpNode::make(init_trip, exit_test.limit, iv_bt); + Node* bol = new BoolNode(cmp_limit, stride_con > 0 ? BoolTest::lt : BoolTest::gt); + insert_loop_limit_check_predicate(init_control->as_IfTrue(), cmp_limit, bol); + } + + if (head->in(LoopNode::LoopBackControl)->Opcode() == Op_SafePoint) { + Node* backedge_sfpt = head->in(LoopNode::LoopBackControl); + if (is_deleteable_safept(backedge_sfpt)) { + lazy_replace(backedge_sfpt, iftrue); + if (loop->_safepts != nullptr) { + loop->_safepts->yank(backedge_sfpt); + } + loop->_tail = iftrue; + } + } + + Node* adjusted_limit = exit_test.limit; if (iv_incr.phi_incr != nullptr) { // If compare points directly to the phi we need to adjust // the compare so that it points to the incr. Limit have @@ -2191,7 +2189,6 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy // Build a canonical trip test. // Clone code, as old values may be in use. -// incr = incr->clone(); Node* incr_clone = increment.incr->clone(); incr_clone->set_req(1,phi); incr_clone->set_req(2, stride.stride); @@ -2238,7 +2235,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy Node *ift2=_igvn.register_new_node_with_optimizer(new IfTrueNode (le)); Node *iff2=_igvn.register_new_node_with_optimizer(new IfFalseNode(le)); - loop->_tail = back_control = ift2; + loop->_tail = back_control = ift2; // XXX: mutation set_loop(ift2, loop); set_loop(iff2, get_loop(iffalse)); @@ -2276,7 +2273,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy // Now setup a new CountedLoopNode to replace the existing LoopNode BaseCountedLoopNode *l = BaseCountedLoopNode::make(entry_control, back_control, iv_bt); - l->set_unswitch_count(x->as_Loop()->unswitch_count()); // Preserve + l->set_unswitch_count(head->as_Loop()->unswitch_count()); // Preserve // The following assert is approximately true, and defines the intention // of can_be_counted_loop. It fails, however, because phase->type // is not yet initialized for this loop and its parts. @@ -2286,7 +2283,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy loop->_head = l; // Fix all data nodes placed at the old loop head. // Uses the lazy-update mechanism of 'get_ctrl'. - lazy_replace( x, l ); + lazy_replace(head, l); set_idom(l, entry_control, dom_depth(entry_control) + 1); if (iv_bt == T_INT && (LoopStripMiningIter == 0 || strip_mine_loop)) { @@ -2345,17 +2342,789 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicTy } #ifndef PRODUCT - if (x->as_Loop()->is_loop_nest_inner_loop() && iv_bt == T_LONG) { + if (head->as_Loop()->is_loop_nest_inner_loop() && iv_bt == T_LONG) { Atomic::inc(&_long_loop_counted_loops); } #endif - if (iv_bt == T_LONG && x->as_Loop()->is_loop_nest_outer_loop()) { + if (iv_bt == T_LONG && head->as_Loop()->is_loop_nest_outer_loop()) { l->mark_loop_nest_outer_loop(); } return true; } +bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { + PhaseGVN* igvn = &_phase->_igvn; + + Node* back_control = _phase->loop_exit_control(_head, _loop); + if (back_control == nullptr) { + return false; + } + + LoopExitTest exit_test = _phase->loop_exit_test(back_control, _loop); + if (exit_test.cmp == nullptr || exit_test.cmp->Opcode() != Op_Cmp(_iv_bt)) { + return false; // Avoid pointer & float & 64-bit compares + } + + // Trip-counter increment must be commutative & associative. + Node* incr = exit_test.incr; // TODO: save to context + if (exit_test.incr->Opcode() == Op_Cast(_iv_bt)) { + incr = incr->in(1); + } + + const LoopIVIncr iv_incr = _phase->loop_iv_incr(incr, _head, _loop); + if (iv_incr.incr == nullptr) { + return false; + } + + CountedLoopNode::TruncatedIncrement + increment = CountedLoopNode::match_incr_with_optional_truncation(iv_incr.incr, _iv_bt); + if (increment.incr == nullptr) { + return false; // Funny increment opcode + } + + assert(increment.incr->Opcode() == Op_Add(_iv_bt), "wrong increment code"); + + const LoopIvStride stride = loop_iv_stride(increment.incr); + if (stride.stride == nullptr) { + return false; + } + + Node* xphi = stride.xphi; // TODO: save to context? + if (xphi->Opcode() == Op_Cast(_iv_bt)) { + xphi = xphi->in(1); + } + + // Stride must be constant + jlong stride_con = stride.stride->get_integer_as_long(_iv_bt); + assert(stride_con != 0, "missed some peephole opt"); + + PhiNode* phi = loop_iv_phi(xphi, iv_incr.phi_incr, _head); + + if (phi == nullptr || + (increment.trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != increment.incr) || + (increment.trunc1 != nullptr && phi->in(LoopNode::LoopBackControl) != increment.trunc1)) { + return false; + } + + const Node* iftrue = back_control; // TODO: save to context? + uint iftrue_op = iftrue->Opcode(); + Node* iff = iftrue->in(0); + BoolNode* test = iff->in(1)->as_Bool(); + + const TypeInteger* limit_t = igvn->type(exit_test.limit)->is_integer(_iv_bt); + if (increment.trunc1 != nullptr) { + // When there is a truncation, we must be sure that after the truncation + // the trip counter will end up higher than the limit, otherwise we are looking + // at an endless loop. Can happen with range checks. + + // Example: + // int i = 0; + // while (true) + // sum + = array[i]; + // i++; + // i = i && 0x7fff; + // } + // + // If the array is shorter than 0x8000 this exits through a AIOOB + // - Counted loop transformation is ok + // If the array is longer then this is an endless loop + // - No transformation can be done. + + const TypeInteger* incr_t = igvn->type(iv_incr.incr)->is_integer(_iv_bt); + if (limit_t->hi_as_long() > incr_t->hi_as_long()) { + // if the limit can have a higher value than the increment (before the0 phi) + return false; + } + } + + Node* init_trip = phi->in(LoopNode::EntryControl); + + // If iv trunc type is smaller than int, check for possible wrap. + if (!TypeInteger::bottom(_iv_bt)->higher_equal(increment.trunc_type)) { + assert(increment.trunc1 != nullptr, "must have found some truncation"); + + // Get a better type for the phi (filtered thru if's) + const TypeInteger* phi_ft = _phase->filtered_type(phi); + + // Can iv take on a value that will wrap? + // + // Ensure iv's limit is not within "stride" of the wrap value. + // + // Example for "short" type + // Truncation ensures value is in the range -32768..32767 (iv_trunc_t) + // If the stride is +10, then the last value of the induction + // variable before the increment (phi_ft->_hi) must be + // <= 32767 - 10 and (phi_ft->_lo) must be >= -32768 to + // ensure no truncation occurs after the increment. + + if (stride_con > 0) { + if (increment.trunc_type->hi_as_long() - phi_ft->hi_as_long() < stride_con || + increment.trunc_type->lo_as_long() > phi_ft->lo_as_long()) { + return false; // truncation may occur + } + } else if (stride_con < 0) { + if (increment.trunc_type->lo_as_long() - phi_ft->lo_as_long() > stride_con || + increment.trunc_type->hi_as_long() < phi_ft->hi_as_long()) { + return false; // truncation may occur + } + } + // No possibility of wrap so truncation can be discarded + // Promote iv type to Int + } else { + assert(increment.trunc1 == nullptr && increment.trunc2 == nullptr, "no truncation for int"); + } + + if (!condition_stride_ok(exit_test.mask, stride_con)) { + return false; + } + + const TypeInteger* init_t = igvn->type(init_trip)->is_integer(_iv_bt); + if (stride_con > 0) { + if (init_t->lo_as_long() > max_signed_integer(_iv_bt) - stride_con) { + return false; // cyclic loop + } + } else { + if (init_t->hi_as_long() < min_signed_integer(_iv_bt) - stride_con) { + return false; // cyclic loop + } + } + + if (iv_incr.phi_incr != nullptr && exit_test.mask != BoolTest::ne) { + // check if there is a possibility of IV overflowing after the first increment + if (stride_con > 0) { + if (init_t->hi_as_long() > max_signed_integer(_iv_bt) - stride_con) { + return false; + } + } else { + if (init_t->lo_as_long() < min_signed_integer(_iv_bt) - stride_con) { + return false; + } + } + } + + // ================================================= + // ---- SUCCESS! Found A Trip-Counted Loop! ----- + // + + if (_head->Opcode() == Op_Region) { + // head has not yet been transformed to Loop or LongCountedLoop. + // This should only happen if we are inside an infinite loop. + // It happens like this: + // build_loop_tree -> do not attach infinite loop and nested loops + // beautify_loops -> does not transform the infinite and nested loops to LoopNode, because not attached yet + // build_loop_tree -> find and attach infinite and nested loops + // counted_loop -> nested Regions are not yet transformed to LoopNodes, we land here + assert(_head->as_Region()->is_in_infinite_subgraph(), + "head can only be a Region and not Loop if inside infinite loop"); + // Come back later when Region is transformed to LoopNode + return false; + } + + assert(_head->Opcode() == Op_Loop || _head->Opcode() == Op_LongCountedLoop, "regular loops only"); + _phase->C->print_method(PHASE_BEFORE_CLOOPS, 3, _head); + + // =================================================== + // We can only convert this loop to a counted loop if we can guarantee that the iv phi will never overflow at runtime. + // This is an implicit assumption taken by some loop optimizations. We therefore must ensure this property at all cost. + // At this point, we've already excluded some trivial cases where an overflow could have been proven statically. + // But even though we cannot prove that an overflow will *not* happen, we still want to speculatively convert this loop + // to a counted loop. This can be achieved by adding additional iv phi overflow checks before the loop. If they fail, + // we trap and resume execution before the loop without having executed any iteration of the loop, yet. + // + // These additional iv phi overflow checks can be inserted as Loop Limit Check Predicates above the Loop Limit Check + // Parse Predicate which captures a JVM state just before the entry of the loop. If there is no such Parse Predicate, + // we cannot generate a Loop Limit Check Predicate and thus cannot speculatively convert the loop to a counted loop. + // + // In the following, we only focus on int loops with stride > 0 to keep things simple. The argumentation and proof + // for stride < 0 is analogously. For long loops, we would replace max_int with max_long. + // + // + // The loop to be converted does not always need to have the often used shape: + // + // i = init + // i = init loop: + // do { ... + // // ... equivalent i+=stride + // i+=stride <==> if (i < limit) + // } while (i < limit); goto loop + // exit: + // ... + // + // where the loop exit check uses the post-incremented iv phi and a '<'-operator. + // + // We could also have '<='-operator (or '>='-operator for negative strides) or use the pre-incremented iv phi value + // in the loop exit check: + // + // i = init + // loop: + // ... + // if (i <= limit) + // i+=stride + // goto loop + // exit: + // ... + // + // Let's define the following terms: + // - iv_pre_i: The pre-incremented iv phi before the i-th iteration. + // - iv_post_i: The post-incremented iv phi after the i-th iteration. + // + // The iv_pre_i and iv_post_i have the following relation: + // iv_pre_i + stride = iv_post_i + // + // When converting a loop to a counted loop, we want to have a canonicalized loop exit check of the form: + // iv_post_i < adjusted_limit + // + // If that is not the case, we need to canonicalize the loop exit check by using different values for adjusted_limit: + // (LE1) iv_post_i < limit: Already canonicalized. We can directly use limit as adjusted_limit. + // -> adjusted_limit = limit. + // (LE2) iv_post_i <= limit: + // iv_post_i < limit + 1 + // -> adjusted limit = limit + 1 + // (LE3) iv_pre_i < limit: + // iv_pre_i + stride < limit + stride + // iv_post_i < limit + stride + // -> adjusted_limit = limit + stride + // (LE4) iv_pre_i <= limit: + // iv_pre_i < limit + 1 + // iv_pre_i + stride < limit + stride + 1 + // iv_post_i < limit + stride + 1 + // -> adjusted_limit = limit + stride + 1 + // + // Note that: + // (AL) limit <= adjusted_limit. + // + // The following loop invariant has to hold for counted loops with n iterations (i.e. loop exit check true after n-th + // loop iteration) and a canonicalized loop exit check to guarantee that no iv_post_i over- or underflows: + // (INV) For i = 1..n, min_int <= iv_post_i <= max_int + // + // To prove (INV), we require the following two conditions/assumptions: + // (i): adjusted_limit - 1 + stride <= max_int + // (ii): init < limit + // + // If we can prove (INV), we know that there can be no over- or underflow of any iv phi value. We prove (INV) by + // induction by assuming (i) and (ii). + // + // Proof by Induction + // ------------------ + // > Base case (i = 1): We show that (INV) holds after the first iteration: + // min_int <= iv_post_1 = init + stride <= max_int + // Proof: + // First, we note that (ii) implies + // (iii) init <= limit - 1 + // max_int >= adjusted_limit - 1 + stride [using (i)] + // >= limit - 1 + stride [using (AL)] + // >= init + stride [using (iii)] + // >= min_int [using stride > 0, no underflow] + // Thus, no overflow happens after the first iteration and (INV) holds for i = 1. + // + // Note that to prove the base case we need (i) and (ii). + // + // > Induction Hypothesis (i = j, j > 1): Assume that (INV) holds after the j-th iteration: + // min_int <= iv_post_j <= max_int + // > Step case (i = j + 1): We show that (INV) also holds after the j+1-th iteration: + // min_int <= iv_post_{j+1} = iv_post_j + stride <= max_int + // Proof: + // If iv_post_j >= adjusted_limit: + // We exit the loop after the j-th iteration, and we don't execute the j+1-th iteration anymore. Thus, there is + // also no iv_{j+1}. Since (INV) holds for iv_j, there is nothing left to prove. + // If iv_post_j < adjusted_limit: + // First, we note that: + // (iv) iv_post_j <= adjusted_limit - 1 + // max_int >= adjusted_limit - 1 + stride [using (i)] + // >= iv_post_j + stride [using (iv)] + // >= min_int [using stride > 0, no underflow] + // + // Note that to prove the step case we only need (i). + // + // Thus, by assuming (i) and (ii), we proved (INV). + // + // + // It is therefore enough to add the following two Loop Limit Check Predicates to check assumptions (i) and (ii): + // + // (1) Loop Limit Check Predicate for (i): + // Using (i): adjusted_limit - 1 + stride <= max_int + // + // This condition is now restated to use limit instead of adjusted_limit: + // + // To prevent an overflow of adjusted_limit -1 + stride itself, we rewrite this check to + // max_int - stride + 1 >= adjusted_limit + // We can merge the two constants into + // canonicalized_correction = stride - 1 + // which gives us + // max_int - canonicalized_correction >= adjusted_limit + // + // To directly use limit instead of adjusted_limit in the predicate condition, we split adjusted_limit into: + // adjusted_limit = limit + limit_correction + // Since stride > 0 and limit_correction <= stride + 1, we can restate this with no over- or underflow into: + // max_int - canonicalized_correction - limit_correction >= limit + // Since canonicalized_correction and limit_correction are both constants, we can replace them with a new constant: + // (v) final_correction = canonicalized_correction + limit_correction + // + // which gives us: + // + // Final predicate condition: + // max_int - final_correction >= limit + // + // However, we need to be careful that (v) does not over- or underflow. + // We know that: + // canonicalized_correction = stride - 1 + // and + // limit_correction <= stride + 1 + // and thus + // canonicalized_correction + limit_correction <= 2 * stride + // To prevent an over- or underflow of (v), we must ensure that + // 2 * stride <= max_int + // which can safely be checked without over- or underflow with + // (vi) stride != min_int AND abs(stride) <= max_int / 2 + // + // We could try to further optimize the cases where (vi) does not hold but given that such large strides are + // very uncommon and the loop would only run for a very few iterations anyway, we simply bail out if (vi) fails. + // + // (2) Loop Limit Check Predicate for (ii): + // Using (ii): init < limit + // + // This Loop Limit Check Predicate is not required if we can prove at compile time that either: + // (2.1) type(init) < type(limit) + // In this case, we know: + // all possible values of init < all possible values of limit + // and we can skip the predicate. + // + // (2.2) init < limit is already checked before (i.e. found as a dominating check) + // In this case, we do not need to re-check the condition and can skip the predicate. + // This is often found for while- and for-loops which have the following shape: + // + // if (init < limit) { // Dominating test. Do not need the Loop Limit Check Predicate below. + // i = init; + // if (init >= limit) { trap(); } // Here we would insert the Loop Limit Check Predicate + // do { + // i += stride; + // } while (i < limit); + // } + // + // (2.3) init + stride <= max_int + // In this case, there is no overflow of the iv phi after the first loop iteration. + // In the proof of the base case above we showed that init + stride <= max_int by using assumption (ii): + // init < limit + // In the proof of the step case above, we did not need (ii) anymore. Therefore, if we already know at + // compile time that init + stride <= max_int then we have trivially proven the base case and that + // there is no overflow of the iv phi after the first iteration. In this case, we don't need to check (ii) + // again and can skip the predicate. + + // Check (vi) and bail out if the stride is too big. + if (stride_con == min_signed_integer(_iv_bt) || (ABS(stride_con) > max_signed_integer(_iv_bt) / 2)) { + return false; + } + + // Accounting for (LE3) and (LE4) where we use pre-incremented phis in the loop exit check. + const jlong limit_correction_for_pre_iv_exit_check = (iv_incr.phi_incr != nullptr) ? stride_con : 0; + + // Accounting for (LE2) and (LE4) where we use <= or >= in the loop exit check. + const bool + includes_limit = (exit_test.mask == BoolTest::le || exit_test.mask == BoolTest::ge); // TODO: save to context + const jlong limit_correction_for_le_ge_exit_check = (includes_limit ? (stride_con > 0 ? 1 : -1) : 0); + + const jlong limit_correction = limit_correction_for_pre_iv_exit_check + limit_correction_for_le_ge_exit_check; + const jlong canonicalized_correction = stride_con + (stride_con > 0 ? -1 : 1); + const jlong final_correction = canonicalized_correction + limit_correction; + + int sov = check_stride_overflow(final_correction, limit_t, _iv_bt); + Node* init_control = _head->in(LoopNode::EntryControl); + + const Predicates predicates(init_control); + const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); + + // TODO: add to context +// bool insert_stride_overflow_limit_check = false; // TODO: replace with sov < 0? +// bool insert_init_trip_limit_check = false; + + if (sov < 0) { + return false; // Bailout: integer overflow is certain. + } + + // If sov == 0, limit's type always satisfies the condition, for + // example, when it is an array length. + if (sov > 0) { + // (1) Loop Limit Check Predicate is required because we could not statically prove that + // limit + final_correction = adjusted_limit - 1 + stride <= max_int + assert(!_head->as_Loop()->is_loop_nest_inner_loop(), "loop was transformed"); + if (!loop_limit_check_predicate_block->has_parse_predicate()) { + // The Loop Limit Check Parse Predicate is not generated if this method trapped here before. +#ifdef ASSERT + if (TraceLoopLimitCheck) { + tty->print("Missing Loop Limit Check Parse Predicate:"); + _loop->dump_head(); + _head->dump(1); + } +#endif + return false; + } + + ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); + if (!_phase->is_dominator(_phase->get_ctrl(exit_test.limit), loop_limit_check_parse_predicate->in(0))) { + return false; + } + + _insert_stride_overflow_limit_check = true; + } + + // (2.3) + const bool init_plus_stride_could_overflow = + (stride_con > 0 && init_t->hi_as_long() > max_signed_integer(_iv_bt) - stride_con) || + (stride_con < 0 && init_t->lo_as_long() < min_signed_integer(_iv_bt) - stride_con); + // (2.1) + const bool init_gte_limit = (stride_con > 0 && init_t->hi_as_long() >= limit_t->lo_as_long()) || + (stride_con < 0 && init_t->lo_as_long() <= limit_t->hi_as_long()); + + if (init_gte_limit && // (2.1) + ((exit_test.mask == BoolTest::ne || init_plus_stride_could_overflow) && // (2.3) + !_phase->has_dominating_loop_limit_check(init_trip, + exit_test.limit, + stride_con, + _iv_bt, + init_control))) { // (2.2) + // (2) Iteration Loop Limit Check Predicate is required because neither (2.1), (2.2), nor (2.3) holds. + // We use the following condition: + // - stride > 0: init < limit + // - stride < 0: init > limit + // + // This predicate is always required if we have a non-equal-operator in the loop exit check (where stride = 1 is + // a requirement). We transform the loop exit check by using a less-than-operator. By doing so, we must always + // check that init < limit. Otherwise, we could have a different number of iterations at runtime. + + if (!loop_limit_check_predicate_block->has_parse_predicate()) { + // The Loop Limit Check Parse Predicate is not generated if this method trapped here before. +#ifdef ASSERT + if (TraceLoopLimitCheck) { + tty->print("Missing Loop Limit Check Parse Predicate:"); + _loop->dump_head(); + _head->dump(1); + } +#endif + return false; + } + + ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); + Node* parse_predicate_entry = loop_limit_check_parse_predicate->in(0); + if (!_phase->is_dominator(_phase->get_ctrl(exit_test.limit), parse_predicate_entry) || + !_phase->is_dominator(_phase->get_ctrl(init_trip), parse_predicate_entry)) { + return false; + } + + _insert_init_trip_limit_check = true; + } + + BoolTest::mask bt = exit_test.mask; // TODO: save bt to context + if (bt == BoolTest::ne) { + // Now we need to canonicalize the loop condition if it is 'ne'. + assert(stride_con == 1 || stride_con == -1, "simple increment only - checked before"); + if (stride_con > 0) { + // 'ne' can be replaced with 'lt' only when init < limit. This is ensured by the inserted predicate above. + bt = BoolTest::lt; + } else { + assert(stride_con < 0, "must be"); + // 'ne' can be replaced with 'gt' only when init > limit. This is ensured by the inserted predicate above. + bt = BoolTest::gt; + } + } + + Node* sfpt = nullptr; // TODO: save sfpt to context + if (_loop->_child == nullptr) { + sfpt = _phase->find_safepoint(back_control, _head, _loop); + } else { + sfpt = iff->in(0); + if (sfpt->Opcode() != Op_SafePoint) { + sfpt = nullptr; + } + } + + if (_head->in(LoopNode::LoopBackControl)->Opcode() == Op_SafePoint) { + if (((_iv_bt == T_INT && LoopStripMiningIter != 0) || + _iv_bt == T_LONG) && + sfpt == nullptr) { + // Leaving the safepoint on the backedge and creating a + // CountedLoop will confuse optimizations. We can't move the + // safepoint around because its jvm state wouldn't match a new + // location. Give up on that loop. + return false; + } + } + +#ifdef ASSERT + // TODO: makes more sense to extract to a separate function? + // Stress and convert to long loop if StressLongCountedLoop flag is set + if (_iv_bt == T_INT && + !_head->as_Loop()->is_loop_nest_inner_loop() && + StressLongCountedLoop > 0 && + increment.trunc1 == nullptr && + _phase->convert_to_long_loop(exit_test.cmp, phi, _loop)) { + return false; + } +#endif + + // TODO: saving only needed variables to context + _limit = exit_test.limit; + _stride_con = stride_con; + _final_correction = final_correction; + _phi = phi; + _phi_incr = iv_incr.phi_incr; + _stride = stride.stride; + _includes_limit = includes_limit; + _bt = bt; + _increment = increment.incr; + _cmp = exit_test.cmp; + _cl_prob = exit_test.cl_prob; + _sfpt = sfpt; + +#ifdef ASSERT + _checked_for_counted_loop = true; +#endif + + return true; +} + +void PhaseIdealLoop::CountedLoopConverter::convert() { +#ifdef ASSERT + assert(_checked_for_counted_loop, "must check for counted loop before conversion"); +#endif + + PhaseIterGVN* igvn = &_phase->_igvn; + Node* init_control = _head->in(LoopNode::EntryControl); + + if (_insert_stride_overflow_limit_check) { + Node* cmp_limit = CmpNode::make(_limit, igvn->integercon((_stride_con > 0 + ? max_signed_integer(_iv_bt) + : min_signed_integer(_iv_bt)) + - _final_correction, _iv_bt), _iv_bt); + Node* bol = new BoolNode(cmp_limit, _stride_con > 0 ? BoolTest::le : BoolTest::ge); + _phase->insert_loop_limit_check_predicate(init_control->as_IfTrue(), cmp_limit, bol); + } + + Node* init_trip = _phi->in(LoopNode::EntryControl); + if (_insert_init_trip_limit_check) { + Node* cmp_limit = CmpNode::make(init_trip, _limit, _iv_bt); + Node* bol = new BoolNode(cmp_limit, _stride_con > 0 ? BoolTest::lt : BoolTest::gt); + _phase->insert_loop_limit_check_predicate(init_control->as_IfTrue(), cmp_limit, bol); + } + + Node* back_control = _phase->loop_exit_control(_head, _loop); + if (_head->in(LoopNode::LoopBackControl)->Opcode() == Op_SafePoint) { + Node* backedge_sfpt = _head->in(LoopNode::LoopBackControl); + if (_phase->is_deleteable_safept(backedge_sfpt)) { + _phase->lazy_replace(backedge_sfpt, back_control); + if (_loop->_safepts != nullptr) { + _loop->_safepts->yank(backedge_sfpt); + } + _loop->_tail = back_control; + } + } + + Node* adjusted_limit = _limit; + if (_phi_incr != nullptr) { + // If compare points directly to the phi we need to adjust + // the compare so that it points to the incr. Limit have + // to be adjusted to keep trip count the same and we + // should avoid int overflow. + // + // i = init; do {} while(i++ < limit); + // is converted to + // i = init; do {} while(++i < limit+1); + // + adjusted_limit = igvn->transform(AddNode::make(_limit, _stride, _iv_bt)); + } + + if (_includes_limit) { + // The limit check guaranties that 'limit <= (max_jint - stride)' so + // we can convert 'i <= limit' to 'i < limit+1' since stride != 0. + // + Node* one = (_stride_con > 0) ? igvn->integercon(1, _iv_bt) : igvn->integercon(-1, _iv_bt); + adjusted_limit = igvn->transform(AddNode::make(adjusted_limit, one, _iv_bt)); + if (_bt == BoolTest::le) + _bt = BoolTest::lt; + else if (_bt == BoolTest::ge) + _bt = BoolTest::gt; + else + ShouldNotReachHere(); + } + _phase->set_subtree_ctrl(adjusted_limit, false); + + // Build a canonical trip test. + // Clone code, as old values may be in use. + Node* incr_clone = _increment->clone(); + incr_clone->set_req(1, _phi); + incr_clone->set_req(2, _stride); + incr_clone = igvn->register_new_node_with_optimizer(incr_clone); + _phase->set_early_ctrl(incr_clone, false); + igvn->rehash_node_delayed(_phi); + _phi->set_req_X(LoopNode::LoopBackControl, incr_clone, igvn); + + // If phi type is more restrictive than Int, raise to + // Int to prevent (almost) infinite recursion in igvn + // which can only handle integer types for constants or minint..maxint. + Node* phi = _phi; + if (!TypeInteger::bottom(_iv_bt)->higher_equal(_phi->bottom_type())) { + Node* nphi = PhiNode::make(_phi->in(0), _phi->in(LoopNode::EntryControl), TypeInteger::bottom(_iv_bt)); + nphi->set_req(LoopNode::LoopBackControl, _phi->in(LoopNode::LoopBackControl)); + nphi = igvn->register_new_node_with_optimizer(nphi); + _phase->set_ctrl(nphi, _phase->get_ctrl(phi)); + igvn->replace_node(_phi, nphi); + phi = nphi->as_Phi(); + } + + Node* iftrue = back_control; + const uint iftrue_op = iftrue->Opcode(); + Node* iff = iftrue->in(0); + BoolNode* test = iff->in(1)->clone()->as_Bool(); + Node* cmp = _cmp->clone(); + + cmp->set_req(1, incr_clone); + cmp->set_req(2, adjusted_limit); + cmp = igvn->register_new_node_with_optimizer(cmp); + _phase->set_ctrl(cmp, iff->in(0)); + + const_cast(&test->_test)->_test = _bt; // Yes, it's const, but it's newly cloned node so it should be fine. + test->set_req(1, cmp); + igvn->register_new_node_with_optimizer(test); + _phase->set_ctrl(test, iff->in(0)); + + // Replace the old IfNode with a new LoopEndNode + Node* lex = igvn->register_new_node_with_optimizer(BaseCountedLoopEndNode::make(iff->in(0), + test, + _cl_prob, + iff->as_If()->_fcnt, + _iv_bt)); + IfNode* le = lex->as_If(); + const uint dd = _phase->dom_depth(iff); + _phase->set_idom(le, le->in(0), dd); // Update dominance for loop exit + _phase->set_loop(le, _loop); + + // Get the loop-exit control + Node* iffalse = iff->as_If()->proj_out(!(iftrue_op == Op_IfTrue)); + +// Need to swap loop-exit and loop-back control? + if (iftrue_op == Op_IfFalse) { + Node* ift2 = igvn->register_new_node_with_optimizer(new IfTrueNode(le)); + Node* iff2 = igvn->register_new_node_with_optimizer(new IfFalseNode(le)); + + _loop->_tail = back_control = ift2; // XXX: mutation + _phase->set_loop(ift2, _loop); + _phase->set_loop(iff2, _phase->get_loop(iffalse)); + + // Lazy update of 'get_ctrl' mechanism. + _phase->lazy_replace(iffalse, iff2); + _phase->lazy_replace(iftrue, ift2); + + // Swap names + iffalse = iff2; + iftrue = ift2; + } else { + igvn->rehash_node_delayed(iffalse); + igvn->rehash_node_delayed(iftrue); + iffalse->set_req_X(0, le, igvn); + iftrue->set_req_X(0, le, igvn); + } + + _phase->set_idom(iftrue, le, dd + 1); + _phase->set_idom(iffalse, le, dd + 1); + assert(iff->outcnt() == 0, "should be dead now"); + _phase->lazy_replace(iff, le); // fix 'get_ctrl' + + Node* entry_control = init_control; + bool strip_mine_loop = _iv_bt == T_INT && + _loop->_child == nullptr && + _sfpt != nullptr && + !_loop->_has_call && + _phase->is_deleteable_safept(_sfpt); + IdealLoopTree* outer_ilt = nullptr; + if (strip_mine_loop) { + outer_ilt = _phase->create_outer_strip_mined_loop(init_control, _loop, + _cl_prob, le->_fcnt, entry_control, + iffalse); + } + + // Now setup a new CountedLoopNode to replace the existing LoopNode + BaseCountedLoopNode* l = BaseCountedLoopNode::make(entry_control, back_control, _iv_bt); + l->set_unswitch_count(_head->as_Loop()->unswitch_count()); // Preserve + // The following assert is approximately true, and defines the intention + // of can_be_counted_loop. It fails, however, because phase->type + // is not yet initialized for this loop and its parts. + //assert(l->can_be_counted_loop(this), "sanity"); + igvn->register_new_node_with_optimizer(l); + _phase->set_loop(l, _loop); + _loop->_head = l; + // Fix all data nodes placed at the old loop head. + // Uses the lazy-update mechanism of 'get_ctrl'. + _phase->lazy_replace(_head, l); + _phase->set_idom(l, entry_control, _phase->dom_depth(entry_control) + 1); + + if (_iv_bt == T_INT && (LoopStripMiningIter == 0 || strip_mine_loop)) { + // Check for immediately preceding SafePoint and remove + if (_sfpt != nullptr && (strip_mine_loop || _phase->is_deleteable_safept(_sfpt))) { + if (strip_mine_loop) { + Node* outer_le = outer_ilt->_tail->in(0); + Node* sfpt_clone = _sfpt->clone(); + sfpt_clone->set_req(0, iffalse); + outer_le->set_req(0, sfpt_clone); + + Node* polladdr = sfpt_clone->in(TypeFunc::Parms); + if (polladdr != nullptr && polladdr->is_Load()) { + // Polling load should be pinned outside inner loop. + Node* new_polladdr = polladdr->clone(); + new_polladdr->set_req(0, iffalse); + igvn->register_new_node_with_optimizer(new_polladdr, polladdr); + _phase->set_ctrl(new_polladdr, iffalse); + sfpt_clone->set_req(TypeFunc::Parms, new_polladdr); + } + // When this code runs, loop bodies have not yet been populated. + const bool body_populated = false; + _phase->register_control(sfpt_clone, outer_ilt, iffalse, body_populated); + _phase->set_idom(outer_le, sfpt_clone, _phase->dom_depth(sfpt_clone)); + } + _phase->lazy_replace(_sfpt, _sfpt->in(TypeFunc::Control)); + if (_loop->_safepts != nullptr) { + _loop->_safepts->yank(_sfpt); + } + } + } + +#ifdef ASSERT + assert(l->is_valid_counted_loop(_iv_bt), "counted loop shape is messed up"); + assert(l == _loop->_head && l->phi() == phi && l->loopexit_or_null() == lex, "" ); +#endif + +#ifndef PRODUCT + if (TraceLoopOpts) { + tty->print("Counted "); + _loop->dump_head(); + } +#endif + + _phase->C->print_method(PHASE_AFTER_CLOOPS, 3, l); + + // Capture bounds of the loop in the induction variable Phi before + // subsequent transformation (iteration splitting) obscures the + // bounds + l->phi()->as_Phi()->set_type(l->phi()->Value(igvn)); + + if (strip_mine_loop) { + l->mark_strip_mined(); + l->verify_strip_mined(1); + outer_ilt->_head->as_Loop()->verify_strip_mined(1); + _loop = outer_ilt; + } + +#ifndef PRODUCT + if (_head->as_Loop()->is_loop_nest_inner_loop() && _iv_bt == T_LONG) { + Atomic::inc(&_long_loop_counted_loops); + } +#endif + + if (_iv_bt == T_LONG && _head->as_Loop()->is_loop_nest_outer_loop()) { + l->mark_loop_nest_outer_loop(); + } +} + + // Check if there is a dominating loop limit check of the form 'init < limit' starting at the loop entry. // If there is one, then we do not need to create an additional Loop Limit Check Predicate. bool PhaseIdealLoop::has_dominating_loop_limit_check(Node* init_trip, Node* limit, const jlong stride_con, diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index c4f9d1ba659a0..a421304dce698 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1290,7 +1290,7 @@ class PhaseIdealLoop : public PhaseTransform { static LoopIvStride loop_iv_stride(const Node* incr); static PhiNode* loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x); - bool is_counted_loop(Node* x, IdealLoopTree*&loop, const BasicType iv_bt); + bool is_counted_loop(Node* head, IdealLoopTree*&loop, const BasicType iv_bt); Node* loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv, Node* outer_phi, Node* inner_head, BasicType bt); bool create_loop_nest(IdealLoopTree* loop, Node_List &old_new); @@ -1304,6 +1304,49 @@ class PhaseIdealLoop : public PhaseTransform { IdealLoopTree* loop, float cl_prob, float le_fcnt, Node*& entry_control, Node*& iffalse); + // FIXME: move to somewhere more sensible + class CountedLoopConverter { + PhaseIdealLoop* _phase; + Node* _head; + IdealLoopTree* _loop; + const BasicType _iv_bt; + + // TODO: better naming for these flags? + bool _insert_stride_overflow_limit_check = false; + bool _insert_init_trip_limit_check = false; + jlong _final_correction; + +#ifdef ASSERT + bool _checked_for_counted_loop = false; +#endif + + Node* _limit; + jlong _stride_con; + Node* _phi; + Node* _phi_incr; + Node* _stride; + bool _includes_limit; + BoolTest::mask _bt; + Node* _increment; + Node* _cmp; + float _cl_prob; + Node* _sfpt; + + public: + CountedLoopConverter(PhaseIdealLoop* phase, Node* head, IdealLoopTree* loop, const BasicType iv_bt) + : _phase(phase), + _head(head), + _loop(loop), + _iv_bt(iv_bt) { + assert(head != nullptr, ""); + assert(loop != nullptr, ""); + assert(iv_bt == T_INT || iv_bt == T_LONG, ""); + } + + bool is_counted_loop(); + void convert(); + }; + Node* exact_limit( IdealLoopTree *loop ); // Return a post-walked LoopNode @@ -1855,10 +1898,6 @@ class PhaseIdealLoop : public PhaseTransform { ConNode* zerocon(BasicType bt); }; -class CountedLoopConverter { - -}; - class AutoNodeBudget : public StackObj { public: From f3305a4088f6e03f9b7bbb72a405458e1fc9d817 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Wed, 2 Apr 2025 15:46:56 -0400 Subject: [PATCH 05/15] fix TestMultiversionRemoveUselessSlowLoop --- src/hotspot/share/opto/loopnode.cpp | 66 +++++++++++++++++++---------- src/hotspot/share/opto/loopnode.hpp | 7 +-- 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index a5adc1959e9a7..446189aa60307 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -1613,18 +1613,44 @@ bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, const Node* phi, const Idea } #endif -//------------------------------is_counted_loop-------------------------------- -bool PhaseIdealLoop::is_counted_loop(Node* head, IdealLoopTree*& loop, const BasicType iv_bt) { + + +//------------------------------try_convert_to_counted_loop-------------------------------- +bool PhaseIdealLoop::try_convert_to_counted_loop(Node* head, IdealLoopTree*& loop, const BasicType iv_bt) { + if (strcmp(Compile::current()->method()->name()->as_utf8(), "testIR") == 0) { + Compile::current()->igv_print_method_to_network("BEFORE"); + } + + bool ret = false; + if (UseNewCode || true) { // FIXME: remove CountedLoopConverter converter(this, head, loop, iv_bt); if (converter.is_counted_loop()) { - converter.convert(); - return true; + loop = converter.convert(); + ret = true; + } else { + ret = false; } + } else if (UseNewCode2) { + CountedLoopConverter converter(this, head, loop, iv_bt); + bool found = converter.is_counted_loop(); + bool expected = try_convert_to_counted_loop_old(head, loop, iv_bt); - return false; + assert(found == expected, "should be the same"); + ret = found; + } else { + ret = try_convert_to_counted_loop_old(head, loop, iv_bt); + } + + + if (strcmp(Compile::current()->method()->name()->as_utf8(), "testIR") == 0) { + Compile::current()->igv_print_method_to_network("AFTER"); } + return ret; +} + +bool PhaseIdealLoop::try_convert_to_counted_loop_old(Node* head, IdealLoopTree*& loop, const BasicType iv_bt) { PhaseGVN *gvn = &_igvn; Node* back_control = loop_exit_control(head, loop); @@ -2005,8 +2031,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* head, IdealLoopTree*& loop, const Bas const Predicates predicates(init_control); const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); - // TODO: add to context - bool insert_stride_overflow_limit_check = false; // TODO: replace with sov < 0? + bool insert_stride_overflow_limit_check = false; bool insert_init_trip_limit_check = false; if (sov < 0) { @@ -2367,7 +2392,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { } // Trip-counter increment must be commutative & associative. - Node* incr = exit_test.incr; // TODO: save to context + Node* incr = exit_test.incr; if (exit_test.incr->Opcode() == Op_Cast(_iv_bt)) { incr = incr->in(1); } @@ -2390,7 +2415,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { return false; } - Node* xphi = stride.xphi; // TODO: save to context? + Node* xphi = stride.xphi; if (xphi->Opcode() == Op_Cast(_iv_bt)) { xphi = xphi->in(1); } @@ -2407,7 +2432,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { return false; } - const Node* iftrue = back_control; // TODO: save to context? + const Node* iftrue = back_control; uint iftrue_op = iftrue->Opcode(); Node* iff = iftrue->in(0); BoolNode* test = iff->in(1)->as_Bool(); @@ -2720,8 +2745,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { const jlong limit_correction_for_pre_iv_exit_check = (iv_incr.phi_incr != nullptr) ? stride_con : 0; // Accounting for (LE2) and (LE4) where we use <= or >= in the loop exit check. - const bool - includes_limit = (exit_test.mask == BoolTest::le || exit_test.mask == BoolTest::ge); // TODO: save to context + const bool includes_limit = (exit_test.mask == BoolTest::le || exit_test.mask == BoolTest::ge); const jlong limit_correction_for_le_ge_exit_check = (includes_limit ? (stride_con > 0 ? 1 : -1) : 0); const jlong limit_correction = limit_correction_for_pre_iv_exit_check + limit_correction_for_le_ge_exit_check; @@ -2734,10 +2758,6 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { const Predicates predicates(init_control); const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); - // TODO: add to context -// bool insert_stride_overflow_limit_check = false; // TODO: replace with sov < 0? -// bool insert_init_trip_limit_check = false; - if (sov < 0) { return false; // Bailout: integer overflow is certain. } @@ -2814,7 +2834,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { _insert_init_trip_limit_check = true; } - BoolTest::mask bt = exit_test.mask; // TODO: save bt to context + BoolTest::mask bt = exit_test.mask; if (bt == BoolTest::ne) { // Now we need to canonicalize the loop condition if it is 'ne'. assert(stride_con == 1 || stride_con == -1, "simple increment only - checked before"); @@ -2828,7 +2848,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { } } - Node* sfpt = nullptr; // TODO: save sfpt to context + Node* sfpt = nullptr; if (_loop->_child == nullptr) { sfpt = _phase->find_safepoint(back_control, _head, _loop); } else { @@ -2883,7 +2903,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { return true; } -void PhaseIdealLoop::CountedLoopConverter::convert() { +IdealLoopTree* PhaseIdealLoop::CountedLoopConverter::convert() { #ifdef ASSERT assert(_checked_for_counted_loop, "must check for counted loop before conversion"); #endif @@ -2974,8 +2994,8 @@ void PhaseIdealLoop::CountedLoopConverter::convert() { Node* iftrue = back_control; const uint iftrue_op = iftrue->Opcode(); Node* iff = iftrue->in(0); - BoolNode* test = iff->in(1)->clone()->as_Bool(); Node* cmp = _cmp->clone(); + BoolNode* test = iff->in(1)->clone()->as_Bool(); cmp->set_req(1, incr_clone); cmp->set_req(2, adjusted_limit); @@ -3122,6 +3142,8 @@ void PhaseIdealLoop::CountedLoopConverter::convert() { if (_iv_bt == T_LONG && _head->as_Loop()->is_loop_nest_outer_loop()) { l->mark_loop_nest_outer_loop(); } + + return _loop; } @@ -4946,7 +4968,7 @@ void IdealLoopTree::counted_loop( PhaseIdealLoop *phase ) { IdealLoopTree* loop = this; if (_head->is_CountedLoop() || - phase->is_counted_loop(_head, loop, T_INT)) { + phase->try_convert_to_counted_loop(_head, loop, T_INT)) { if (LoopStripMiningIter == 0 || _head->as_CountedLoop()->is_strip_mined()) { // Indicate we do not need a safepoint here @@ -4960,7 +4982,7 @@ void IdealLoopTree::counted_loop( PhaseIdealLoop *phase ) { // Look for induction variables phase->replace_parallel_iv(this); } else if (_head->is_LongCountedLoop() || - phase->is_counted_loop(_head, loop, T_LONG)) { + phase->try_convert_to_counted_loop(_head, loop, T_LONG)) { remove_safepoints(phase, true); } else { assert(!_head->is_Loop() || !_head->as_Loop()->is_loop_nest_inner_loop(), "transformation to counted loop should not fail"); diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index a421304dce698..3f1cbd3a02ec9 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1290,7 +1290,8 @@ class PhaseIdealLoop : public PhaseTransform { static LoopIvStride loop_iv_stride(const Node* incr); static PhiNode* loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x); - bool is_counted_loop(Node* head, IdealLoopTree*&loop, const BasicType iv_bt); + bool try_convert_to_counted_loop_old(Node* head, IdealLoopTree*& loop, const BasicType iv_bt); + bool try_convert_to_counted_loop(Node* head, IdealLoopTree*&loop, const BasicType iv_bt); Node* loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv, Node* outer_phi, Node* inner_head, BasicType bt); bool create_loop_nest(IdealLoopTree* loop, Node_List &old_new); @@ -1314,7 +1315,6 @@ class PhaseIdealLoop : public PhaseTransform { // TODO: better naming for these flags? bool _insert_stride_overflow_limit_check = false; bool _insert_init_trip_limit_check = false; - jlong _final_correction; #ifdef ASSERT bool _checked_for_counted_loop = false; @@ -1331,6 +1331,7 @@ class PhaseIdealLoop : public PhaseTransform { Node* _cmp; float _cl_prob; Node* _sfpt; + jlong _final_correction; public: CountedLoopConverter(PhaseIdealLoop* phase, Node* head, IdealLoopTree* loop, const BasicType iv_bt) @@ -1344,7 +1345,7 @@ class PhaseIdealLoop : public PhaseTransform { } bool is_counted_loop(); - void convert(); + IdealLoopTree* convert(); }; Node* exact_limit( IdealLoopTree *loop ); From f3496935281fb02ed0fe78f0cc92defa08e3f400 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Wed, 2 Apr 2025 15:56:56 -0400 Subject: [PATCH 06/15] remove debug lines --- src/hotspot/share/opto/loopnode.cpp | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 446189aa60307..0aff76d018fe5 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -1617,37 +1617,17 @@ bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, const Node* phi, const Idea //------------------------------try_convert_to_counted_loop-------------------------------- bool PhaseIdealLoop::try_convert_to_counted_loop(Node* head, IdealLoopTree*& loop, const BasicType iv_bt) { - if (strcmp(Compile::current()->method()->name()->as_utf8(), "testIR") == 0) { - Compile::current()->igv_print_method_to_network("BEFORE"); - } - - bool ret = false; - if (UseNewCode || true) { // FIXME: remove CountedLoopConverter converter(this, head, loop, iv_bt); if (converter.is_counted_loop()) { loop = converter.convert(); - ret = true; + return true; } else { - ret = false; + return false; } - } else if (UseNewCode2) { - CountedLoopConverter converter(this, head, loop, iv_bt); - bool found = converter.is_counted_loop(); - bool expected = try_convert_to_counted_loop_old(head, loop, iv_bt); - - assert(found == expected, "should be the same"); - ret = found; - } else { - ret = try_convert_to_counted_loop_old(head, loop, iv_bt); - } - - - if (strcmp(Compile::current()->method()->name()->as_utf8(), "testIR") == 0) { - Compile::current()->igv_print_method_to_network("AFTER"); } - return ret; + return try_convert_to_counted_loop_old(head, loop, iv_bt); } bool PhaseIdealLoop::try_convert_to_counted_loop_old(Node* head, IdealLoopTree*& loop, const BasicType iv_bt) { From c2219865bec922bc5512a325bfdc192785b2ac74 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Fri, 4 Apr 2025 13:43:59 -0400 Subject: [PATCH 07/15] extract stress_long_counted_loop() --- src/hotspot/share/opto/loopnode.cpp | 871 ++++------------------------ src/hotspot/share/opto/loopnode.hpp | 6 + 2 files changed, 133 insertions(+), 744 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 0aff76d018fe5..bde78efa124a5 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -1617,745 +1617,20 @@ bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, const Node* phi, const Idea //------------------------------try_convert_to_counted_loop-------------------------------- bool PhaseIdealLoop::try_convert_to_counted_loop(Node* head, IdealLoopTree*& loop, const BasicType iv_bt) { - if (UseNewCode || true) { // FIXME: remove - CountedLoopConverter converter(this, head, loop, iv_bt); - if (converter.is_counted_loop()) { - loop = converter.convert(); - return true; - } else { - return false; - } - } - - return try_convert_to_counted_loop_old(head, loop, iv_bt); -} - -bool PhaseIdealLoop::try_convert_to_counted_loop_old(Node* head, IdealLoopTree*& loop, const BasicType iv_bt) { - PhaseGVN *gvn = &_igvn; - - Node* back_control = loop_exit_control(head, loop); - if (back_control == nullptr) { - return false; - } - - LoopExitTest exit_test = loop_exit_test(back_control, loop); - if (exit_test.cmp == nullptr || exit_test.cmp->Opcode() != Op_Cmp(iv_bt)) { - return false; // Avoid pointer & float & 64-bit compares - } - - // Trip-counter increment must be commutative & associative. - Node* incr = exit_test.incr; - if (exit_test.incr->Opcode() == Op_Cast(iv_bt)) { - incr = incr->in(1); - } - - const LoopIVIncr iv_incr = loop_iv_incr(incr, head, loop); - if (iv_incr.incr == nullptr) { - return false; - } - - CountedLoopNode::TruncatedIncrement increment = CountedLoopNode::match_incr_with_optional_truncation(iv_incr.incr, iv_bt); - if (increment.incr == nullptr) { - return false; // Funny increment opcode - } - assert(increment.incr->Opcode() == Op_Add(iv_bt), "wrong increment code"); - - const LoopIvStride stride = loop_iv_stride(increment.incr); - - if (stride.stride == nullptr) { - return false; - } - - Node* xphi = stride.xphi; - if (xphi->Opcode() == Op_Cast(iv_bt)) { - xphi = xphi->in(1); - } - - // Stride must be constant - jlong stride_con = stride.stride->get_integer_as_long(iv_bt); - assert(stride_con != 0, "missed some peephole opt"); - - PhiNode* phi = loop_iv_phi(xphi, iv_incr.phi_incr, head); - - if (phi == nullptr || - (increment.trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != increment.incr) || - (increment.trunc1 != nullptr && phi->in(LoopNode::LoopBackControl) != increment.trunc1)) { - return false; - } - - Node* iftrue = back_control; - uint iftrue_op = iftrue->Opcode(); - Node* iff = iftrue->in(0); - BoolNode* test = iff->in(1)->as_Bool(); - - const TypeInteger* limit_t = gvn->type(exit_test.limit)->is_integer(iv_bt); - if (increment.trunc1 != nullptr) { - // When there is a truncation, we must be sure that after the truncation - // the trip counter will end up higher than the limit, otherwise we are looking - // at an endless loop. Can happen with range checks. - - // Example: - // int i = 0; - // while (true) - // sum + = array[i]; - // i++; - // i = i && 0x7fff; - // } - // - // If the array is shorter than 0x8000 this exits through a AIOOB - // - Counted loop transformation is ok - // If the array is longer then this is an endless loop - // - No transformation can be done. - - const TypeInteger* incr_t = gvn->type(iv_incr.incr)->is_integer(iv_bt); - if (limit_t->hi_as_long() > incr_t->hi_as_long()) { - // if the limit can have a higher value than the increment (before the0 phi) - return false; - } - } - - Node *init_trip = phi->in(LoopNode::EntryControl); - - // If iv trunc type is smaller than int, check for possible wrap. - if (!TypeInteger::bottom(iv_bt)->higher_equal(increment.trunc_type)) { - assert(increment.trunc1 != nullptr, "must have found some truncation"); - - // Get a better type for the phi (filtered thru if's) - const TypeInteger* phi_ft = filtered_type(phi); - - // Can iv take on a value that will wrap? - // - // Ensure iv's limit is not within "stride" of the wrap value. - // - // Example for "short" type - // Truncation ensures value is in the range -32768..32767 (iv_trunc_t) - // If the stride is +10, then the last value of the induction - // variable before the increment (phi_ft->_hi) must be - // <= 32767 - 10 and (phi_ft->_lo) must be >= -32768 to - // ensure no truncation occurs after the increment. - - if (stride_con > 0) { - if (increment.trunc_type->hi_as_long() - phi_ft->hi_as_long() < stride_con || - increment.trunc_type->lo_as_long() > phi_ft->lo_as_long()) { - return false; // truncation may occur - } - } else if (stride_con < 0) { - if (increment.trunc_type->lo_as_long() - phi_ft->lo_as_long() > stride_con || - increment.trunc_type->hi_as_long() < phi_ft->hi_as_long()) { - return false; // truncation may occur - } - } - // No possibility of wrap so truncation can be discarded - // Promote iv type to Int - } else { - assert(increment.trunc1 == nullptr && increment.trunc2 == nullptr, "no truncation for int"); - } - - if (!condition_stride_ok(exit_test.mask, stride_con)) { - return false; - } - - const TypeInteger* init_t = gvn->type(init_trip)->is_integer(iv_bt); - - if (stride_con > 0) { - if (init_t->lo_as_long() > max_signed_integer(iv_bt) - stride_con) { - return false; // cyclic loop - } - } else { - if (init_t->hi_as_long() < min_signed_integer(iv_bt) - stride_con) { - return false; // cyclic loop - } - } - - if (iv_incr.phi_incr != nullptr && exit_test.mask != BoolTest::ne) { - // check if there is a possibility of IV overflowing after the first increment - if (stride_con > 0) { - if (init_t->hi_as_long() > max_signed_integer(iv_bt) - stride_con) { - return false; - } - } else { - if (init_t->lo_as_long() < min_signed_integer(iv_bt) - stride_con) { - return false; - } - } - } - - // ================================================= - // ---- SUCCESS! Found A Trip-Counted Loop! ----- - // - - if (head->Opcode() == Op_Region) { - // head has not yet been transformed to Loop or LongCountedLoop. - // This should only happen if we are inside an infinite loop. - // It happens like this: - // build_loop_tree -> do not attach infinite loop and nested loops - // beautify_loops -> does not transform the infinite and nested loops to LoopNode, because not attached yet - // build_loop_tree -> find and attach infinite and nested loops - // counted_loop -> nested Regions are not yet transformed to LoopNodes, we land here - assert(head->as_Region()->is_in_infinite_subgraph(), - "head can only be a Region and not Loop if inside infinite loop"); - // Come back later when Region is transformed to LoopNode - return false; - } - - assert(head->Opcode() == Op_Loop || head->Opcode() == Op_LongCountedLoop, "regular loops only"); - C->print_method(PHASE_BEFORE_CLOOPS, 3, head); - - // =================================================== - // We can only convert this loop to a counted loop if we can guarantee that the iv phi will never overflow at runtime. - // This is an implicit assumption taken by some loop optimizations. We therefore must ensure this property at all cost. - // At this point, we've already excluded some trivial cases where an overflow could have been proven statically. - // But even though we cannot prove that an overflow will *not* happen, we still want to speculatively convert this loop - // to a counted loop. This can be achieved by adding additional iv phi overflow checks before the loop. If they fail, - // we trap and resume execution before the loop without having executed any iteration of the loop, yet. - // - // These additional iv phi overflow checks can be inserted as Loop Limit Check Predicates above the Loop Limit Check - // Parse Predicate which captures a JVM state just before the entry of the loop. If there is no such Parse Predicate, - // we cannot generate a Loop Limit Check Predicate and thus cannot speculatively convert the loop to a counted loop. - // - // In the following, we only focus on int loops with stride > 0 to keep things simple. The argumentation and proof - // for stride < 0 is analogously. For long loops, we would replace max_int with max_long. - // - // - // The loop to be converted does not always need to have the often used shape: - // - // i = init - // i = init loop: - // do { ... - // // ... equivalent i+=stride - // i+=stride <==> if (i < limit) - // } while (i < limit); goto loop - // exit: - // ... - // - // where the loop exit check uses the post-incremented iv phi and a '<'-operator. - // - // We could also have '<='-operator (or '>='-operator for negative strides) or use the pre-incremented iv phi value - // in the loop exit check: - // - // i = init - // loop: - // ... - // if (i <= limit) - // i+=stride - // goto loop - // exit: - // ... - // - // Let's define the following terms: - // - iv_pre_i: The pre-incremented iv phi before the i-th iteration. - // - iv_post_i: The post-incremented iv phi after the i-th iteration. - // - // The iv_pre_i and iv_post_i have the following relation: - // iv_pre_i + stride = iv_post_i - // - // When converting a loop to a counted loop, we want to have a canonicalized loop exit check of the form: - // iv_post_i < adjusted_limit - // - // If that is not the case, we need to canonicalize the loop exit check by using different values for adjusted_limit: - // (LE1) iv_post_i < limit: Already canonicalized. We can directly use limit as adjusted_limit. - // -> adjusted_limit = limit. - // (LE2) iv_post_i <= limit: - // iv_post_i < limit + 1 - // -> adjusted limit = limit + 1 - // (LE3) iv_pre_i < limit: - // iv_pre_i + stride < limit + stride - // iv_post_i < limit + stride - // -> adjusted_limit = limit + stride - // (LE4) iv_pre_i <= limit: - // iv_pre_i < limit + 1 - // iv_pre_i + stride < limit + stride + 1 - // iv_post_i < limit + stride + 1 - // -> adjusted_limit = limit + stride + 1 - // - // Note that: - // (AL) limit <= adjusted_limit. - // - // The following loop invariant has to hold for counted loops with n iterations (i.e. loop exit check true after n-th - // loop iteration) and a canonicalized loop exit check to guarantee that no iv_post_i over- or underflows: - // (INV) For i = 1..n, min_int <= iv_post_i <= max_int - // - // To prove (INV), we require the following two conditions/assumptions: - // (i): adjusted_limit - 1 + stride <= max_int - // (ii): init < limit - // - // If we can prove (INV), we know that there can be no over- or underflow of any iv phi value. We prove (INV) by - // induction by assuming (i) and (ii). - // - // Proof by Induction - // ------------------ - // > Base case (i = 1): We show that (INV) holds after the first iteration: - // min_int <= iv_post_1 = init + stride <= max_int - // Proof: - // First, we note that (ii) implies - // (iii) init <= limit - 1 - // max_int >= adjusted_limit - 1 + stride [using (i)] - // >= limit - 1 + stride [using (AL)] - // >= init + stride [using (iii)] - // >= min_int [using stride > 0, no underflow] - // Thus, no overflow happens after the first iteration and (INV) holds for i = 1. - // - // Note that to prove the base case we need (i) and (ii). - // - // > Induction Hypothesis (i = j, j > 1): Assume that (INV) holds after the j-th iteration: - // min_int <= iv_post_j <= max_int - // > Step case (i = j + 1): We show that (INV) also holds after the j+1-th iteration: - // min_int <= iv_post_{j+1} = iv_post_j + stride <= max_int - // Proof: - // If iv_post_j >= adjusted_limit: - // We exit the loop after the j-th iteration, and we don't execute the j+1-th iteration anymore. Thus, there is - // also no iv_{j+1}. Since (INV) holds for iv_j, there is nothing left to prove. - // If iv_post_j < adjusted_limit: - // First, we note that: - // (iv) iv_post_j <= adjusted_limit - 1 - // max_int >= adjusted_limit - 1 + stride [using (i)] - // >= iv_post_j + stride [using (iv)] - // >= min_int [using stride > 0, no underflow] - // - // Note that to prove the step case we only need (i). - // - // Thus, by assuming (i) and (ii), we proved (INV). - // - // - // It is therefore enough to add the following two Loop Limit Check Predicates to check assumptions (i) and (ii): - // - // (1) Loop Limit Check Predicate for (i): - // Using (i): adjusted_limit - 1 + stride <= max_int - // - // This condition is now restated to use limit instead of adjusted_limit: - // - // To prevent an overflow of adjusted_limit -1 + stride itself, we rewrite this check to - // max_int - stride + 1 >= adjusted_limit - // We can merge the two constants into - // canonicalized_correction = stride - 1 - // which gives us - // max_int - canonicalized_correction >= adjusted_limit - // - // To directly use limit instead of adjusted_limit in the predicate condition, we split adjusted_limit into: - // adjusted_limit = limit + limit_correction - // Since stride > 0 and limit_correction <= stride + 1, we can restate this with no over- or underflow into: - // max_int - canonicalized_correction - limit_correction >= limit - // Since canonicalized_correction and limit_correction are both constants, we can replace them with a new constant: - // (v) final_correction = canonicalized_correction + limit_correction - // - // which gives us: - // - // Final predicate condition: - // max_int - final_correction >= limit - // - // However, we need to be careful that (v) does not over- or underflow. - // We know that: - // canonicalized_correction = stride - 1 - // and - // limit_correction <= stride + 1 - // and thus - // canonicalized_correction + limit_correction <= 2 * stride - // To prevent an over- or underflow of (v), we must ensure that - // 2 * stride <= max_int - // which can safely be checked without over- or underflow with - // (vi) stride != min_int AND abs(stride) <= max_int / 2 - // - // We could try to further optimize the cases where (vi) does not hold but given that such large strides are - // very uncommon and the loop would only run for a very few iterations anyway, we simply bail out if (vi) fails. - // - // (2) Loop Limit Check Predicate for (ii): - // Using (ii): init < limit - // - // This Loop Limit Check Predicate is not required if we can prove at compile time that either: - // (2.1) type(init) < type(limit) - // In this case, we know: - // all possible values of init < all possible values of limit - // and we can skip the predicate. - // - // (2.2) init < limit is already checked before (i.e. found as a dominating check) - // In this case, we do not need to re-check the condition and can skip the predicate. - // This is often found for while- and for-loops which have the following shape: - // - // if (init < limit) { // Dominating test. Do not need the Loop Limit Check Predicate below. - // i = init; - // if (init >= limit) { trap(); } // Here we would insert the Loop Limit Check Predicate - // do { - // i += stride; - // } while (i < limit); - // } - // - // (2.3) init + stride <= max_int - // In this case, there is no overflow of the iv phi after the first loop iteration. - // In the proof of the base case above we showed that init + stride <= max_int by using assumption (ii): - // init < limit - // In the proof of the step case above, we did not need (ii) anymore. Therefore, if we already know at - // compile time that init + stride <= max_int then we have trivially proven the base case and that - // there is no overflow of the iv phi after the first iteration. In this case, we don't need to check (ii) - // again and can skip the predicate. - - // Check (vi) and bail out if the stride is too big. - if (stride_con == min_signed_integer(iv_bt) || (ABS(stride_con) > max_signed_integer(iv_bt) / 2)) { - return false; - } - - // Accounting for (LE3) and (LE4) where we use pre-incremented phis in the loop exit check. - const jlong limit_correction_for_pre_iv_exit_check = (iv_incr.phi_incr != nullptr) ? stride_con : 0; - - // Accounting for (LE2) and (LE4) where we use <= or >= in the loop exit check. - const bool - includes_limit = (exit_test.mask == BoolTest::le || exit_test.mask == BoolTest::ge); // TODO: save to context - const jlong limit_correction_for_le_ge_exit_check = (includes_limit ? (stride_con > 0 ? 1 : -1) : 0); - - const jlong limit_correction = limit_correction_for_pre_iv_exit_check + limit_correction_for_le_ge_exit_check; - const jlong canonicalized_correction = stride_con + (stride_con > 0 ? -1 : 1); - const jlong final_correction = canonicalized_correction + limit_correction; - - int sov = check_stride_overflow(final_correction, limit_t, iv_bt); - Node* init_control = head->in(LoopNode::EntryControl); - - const Predicates predicates(init_control); - const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); - - bool insert_stride_overflow_limit_check = false; - bool insert_init_trip_limit_check = false; - - if (sov < 0) { - return false; // Bailout: integer overflow is certain. - } - - // If sov == 0, limit's type always satisfies the condition, for - // example, when it is an array length. - if (sov > 0) { - // (1) Loop Limit Check Predicate is required because we could not statically prove that - // limit + final_correction = adjusted_limit - 1 + stride <= max_int - assert(!head->as_Loop()->is_loop_nest_inner_loop(), "loop was transformed"); - if (!loop_limit_check_predicate_block->has_parse_predicate()) { - // The Loop Limit Check Parse Predicate is not generated if this method trapped here before. -#ifdef ASSERT - if (TraceLoopLimitCheck) { - tty->print("Missing Loop Limit Check Parse Predicate:"); - loop->dump_head(); - head->dump(1); - } -#endif - return false; - } - - ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); - if (!is_dominator(get_ctrl(exit_test.limit), loop_limit_check_parse_predicate->in(0))) { - return false; - } - - insert_stride_overflow_limit_check = true; - } - - // (2.3) - const bool init_plus_stride_could_overflow = - (stride_con > 0 && init_t->hi_as_long() > max_signed_integer(iv_bt) - stride_con) || - (stride_con < 0 && init_t->lo_as_long() < min_signed_integer(iv_bt) - stride_con); - // (2.1) - const bool init_gte_limit = (stride_con > 0 && init_t->hi_as_long() >= limit_t->lo_as_long()) || - (stride_con < 0 && init_t->lo_as_long() <= limit_t->hi_as_long()); - - if (init_gte_limit && // (2.1) - ((exit_test.mask == BoolTest::ne || init_plus_stride_could_overflow) && // (2.3) - !has_dominating_loop_limit_check(init_trip, exit_test.limit, stride_con, iv_bt, init_control))) { // (2.2) - // (2) Iteration Loop Limit Check Predicate is required because neither (2.1), (2.2), nor (2.3) holds. - // We use the following condition: - // - stride > 0: init < limit - // - stride < 0: init > limit - // - // This predicate is always required if we have a non-equal-operator in the loop exit check (where stride = 1 is - // a requirement). We transform the loop exit check by using a less-than-operator. By doing so, we must always - // check that init < limit. Otherwise, we could have a different number of iterations at runtime. - - if (!loop_limit_check_predicate_block->has_parse_predicate()) { - // The Loop Limit Check Parse Predicate is not generated if this method trapped here before. + CountedLoopConverter converter(this, head, loop, iv_bt); + if (converter.is_counted_loop()) { #ifdef ASSERT - if (TraceLoopLimitCheck) { - tty->print("Missing Loop Limit Check Parse Predicate:"); - loop->dump_head(); - head->dump(1); - } -#endif - return false; - } - - ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); - Node* parse_predicate_entry = loop_limit_check_parse_predicate->in(0); - if (!is_dominator(get_ctrl(exit_test.limit), parse_predicate_entry) || - !is_dominator(get_ctrl(init_trip), parse_predicate_entry)) { - return false; - } - - insert_init_trip_limit_check = true; - } - - BoolTest::mask bt = exit_test.mask; // TODO: need to save - if (bt == BoolTest::ne) { - // Now we need to canonicalize the loop condition if it is 'ne'. - assert(stride_con == 1 || stride_con == -1, "simple increment only - checked before"); - if (stride_con > 0) { - // 'ne' can be replaced with 'lt' only when init < limit. This is ensured by the inserted predicate above. - bt = BoolTest::lt; - } else { - assert(stride_con < 0, "must be"); - // 'ne' can be replaced with 'gt' only when init > limit. This is ensured by the inserted predicate above. - bt = BoolTest::gt; - } - } - - Node* sfpt = nullptr; // TODO: save sfpt - if (loop->_child == nullptr) { - sfpt = find_safepoint(back_control, head, loop); - } else { - sfpt = iff->in(0); - if (sfpt->Opcode() != Op_SafePoint) { - sfpt = nullptr; - } - } - - if (head->in(LoopNode::LoopBackControl)->Opcode() == Op_SafePoint) { - if (((iv_bt == T_INT && LoopStripMiningIter != 0) || - iv_bt == T_LONG) && - sfpt == nullptr) { - // Leaving the safepoint on the backedge and creating a - // CountedLoop will confuse optimizations. We can't move the - // safepoint around because its jvm state wouldn't match a new - // location. Give up on that loop. + // Stress by converting int counted loops to long counted loops + if (converter.should_stress_long_counted_loop() && converter.stress_long_counted_loop()) { return false; } - } - - -#ifdef ASSERT - // Stress and convert to long loop if StressLongCountedLoop flag is set - if (iv_bt == T_INT && - !head->as_Loop()->is_loop_nest_inner_loop() && - StressLongCountedLoop > 0 && - increment.trunc1 == nullptr && - convert_to_long_loop(exit_test.cmp, phi, loop)) { - return false; - } -#endif - - // return true; - - // TODO: extract to separate function? - if (insert_stride_overflow_limit_check) { - Node* cmp_limit = CmpNode::make(exit_test.limit, _igvn.integercon((stride_con > 0 - ? max_signed_integer(iv_bt) - : min_signed_integer(iv_bt)) - - final_correction, iv_bt), iv_bt); - Node* bol = new BoolNode(cmp_limit, stride_con > 0 ? BoolTest::le : BoolTest::ge); - insert_loop_limit_check_predicate(init_control->as_IfTrue(), cmp_limit, bol); - } - - if (insert_init_trip_limit_check) { - Node* cmp_limit = CmpNode::make(init_trip, exit_test.limit, iv_bt); - Node* bol = new BoolNode(cmp_limit, stride_con > 0 ? BoolTest::lt : BoolTest::gt); - insert_loop_limit_check_predicate(init_control->as_IfTrue(), cmp_limit, bol); - } - - if (head->in(LoopNode::LoopBackControl)->Opcode() == Op_SafePoint) { - Node* backedge_sfpt = head->in(LoopNode::LoopBackControl); - if (is_deleteable_safept(backedge_sfpt)) { - lazy_replace(backedge_sfpt, iftrue); - if (loop->_safepts != nullptr) { - loop->_safepts->yank(backedge_sfpt); - } - loop->_tail = iftrue; - } - } - - Node* adjusted_limit = exit_test.limit; - if (iv_incr.phi_incr != nullptr) { - // If compare points directly to the phi we need to adjust - // the compare so that it points to the incr. Limit have - // to be adjusted to keep trip count the same and we - // should avoid int overflow. - // - // i = init; do {} while(i++ < limit); - // is converted to - // i = init; do {} while(++i < limit+1); - // - adjusted_limit = gvn->transform(AddNode::make(exit_test.limit, stride.stride, iv_bt)); - } - - if (includes_limit) { - // The limit check guaranties that 'limit <= (max_jint - stride)' so - // we can convert 'i <= limit' to 'i < limit+1' since stride != 0. - // - Node* one = (stride_con > 0) ? gvn->integercon( 1, iv_bt) : gvn->integercon(-1, iv_bt); - adjusted_limit = gvn->transform(AddNode::make(adjusted_limit, one, iv_bt)); - if (bt == BoolTest::le) - bt = BoolTest::lt; - else if (bt == BoolTest::ge) - bt = BoolTest::gt; - else - ShouldNotReachHere(); - } - set_subtree_ctrl(adjusted_limit, false); - - // Build a canonical trip test. - // Clone code, as old values may be in use. - Node* incr_clone = increment.incr->clone(); - incr_clone->set_req(1,phi); - incr_clone->set_req(2, stride.stride); - incr_clone = _igvn.register_new_node_with_optimizer(incr_clone); - set_early_ctrl(incr_clone, false); - _igvn.rehash_node_delayed(phi); - phi->set_req_X( LoopNode::LoopBackControl, incr_clone, &_igvn ); - - // If phi type is more restrictive than Int, raise to - // Int to prevent (almost) infinite recursion in igvn - // which can only handle integer types for constants or minint..maxint. - if (!TypeInteger::bottom(iv_bt)->higher_equal(phi->bottom_type())) { - Node* nphi = PhiNode::make(phi->in(0), phi->in(LoopNode::EntryControl), TypeInteger::bottom(iv_bt)); - nphi->set_req(LoopNode::LoopBackControl, phi->in(LoopNode::LoopBackControl)); - nphi = _igvn.register_new_node_with_optimizer(nphi); - set_ctrl(nphi, get_ctrl(phi)); - _igvn.replace_node(phi, nphi); - phi = nphi->as_Phi(); - } - Node* cmp = exit_test.cmp->clone(); // TODO: need to save - cmp->set_req(1,incr_clone); - cmp->set_req(2, adjusted_limit); - cmp = _igvn.register_new_node_with_optimizer(cmp); - set_ctrl(cmp, iff->in(0)); - - test = test->clone()->as_Bool(); - (*(BoolTest*)&test->_test)._test = bt; - test->set_req(1,cmp); - _igvn.register_new_node_with_optimizer(test); - set_ctrl(test, iff->in(0)); - - // Replace the old IfNode with a new LoopEndNode - Node *lex = _igvn.register_new_node_with_optimizer(BaseCountedLoopEndNode::make(iff->in(0), test, exit_test.cl_prob, iff->as_If()->_fcnt, iv_bt)); - IfNode *le = lex->as_If(); - uint dd = dom_depth(iff); - set_idom(le, le->in(0), dd); // Update dominance for loop exit - set_loop(le, loop); - - // Get the loop-exit control - Node *iffalse = iff->as_If()->proj_out(!(iftrue_op == Op_IfTrue)); - - // Need to swap loop-exit and loop-back control? - if (iftrue_op == Op_IfFalse) { - Node *ift2=_igvn.register_new_node_with_optimizer(new IfTrueNode (le)); - Node *iff2=_igvn.register_new_node_with_optimizer(new IfFalseNode(le)); - - loop->_tail = back_control = ift2; // XXX: mutation - set_loop(ift2, loop); - set_loop(iff2, get_loop(iffalse)); - - // Lazy update of 'get_ctrl' mechanism. - lazy_replace(iffalse, iff2); - lazy_replace(iftrue, ift2); - - // Swap names - iffalse = iff2; - iftrue = ift2; - } else { - _igvn.rehash_node_delayed(iffalse); - _igvn.rehash_node_delayed(iftrue); - iffalse->set_req_X( 0, le, &_igvn ); - iftrue ->set_req_X( 0, le, &_igvn ); - } - - set_idom(iftrue, le, dd+1); - set_idom(iffalse, le, dd+1); - assert(iff->outcnt() == 0, "should be dead now"); - lazy_replace( iff, le ); // fix 'get_ctrl' - - Node* entry_control = init_control; - bool strip_mine_loop = iv_bt == T_INT && - loop->_child == nullptr && - sfpt != nullptr && - !loop->_has_call && - is_deleteable_safept(sfpt); - IdealLoopTree* outer_ilt = nullptr; - if (strip_mine_loop) { - outer_ilt = create_outer_strip_mined_loop(init_control, loop, - exit_test.cl_prob, le->_fcnt, entry_control, - iffalse); - } - - // Now setup a new CountedLoopNode to replace the existing LoopNode - BaseCountedLoopNode *l = BaseCountedLoopNode::make(entry_control, back_control, iv_bt); - l->set_unswitch_count(head->as_Loop()->unswitch_count()); // Preserve - // The following assert is approximately true, and defines the intention - // of can_be_counted_loop. It fails, however, because phase->type - // is not yet initialized for this loop and its parts. - //assert(l->can_be_counted_loop(this), "sanity"); - _igvn.register_new_node_with_optimizer(l); - set_loop(l, loop); - loop->_head = l; - // Fix all data nodes placed at the old loop head. - // Uses the lazy-update mechanism of 'get_ctrl'. - lazy_replace(head, l); - set_idom(l, entry_control, dom_depth(entry_control) + 1); - - if (iv_bt == T_INT && (LoopStripMiningIter == 0 || strip_mine_loop)) { - // Check for immediately preceding SafePoint and remove - if (sfpt != nullptr && (strip_mine_loop || is_deleteable_safept(sfpt))) { - if (strip_mine_loop) { - Node* outer_le = outer_ilt->_tail->in(0); - Node* sfpt_clone = sfpt->clone(); - sfpt_clone->set_req(0, iffalse); - outer_le->set_req(0, sfpt_clone); - - Node* polladdr = sfpt_clone->in(TypeFunc::Parms); - if (polladdr != nullptr && polladdr->is_Load()) { - // Polling load should be pinned outside inner loop. - Node* new_polladdr = polladdr->clone(); - new_polladdr->set_req(0, iffalse); - _igvn.register_new_node_with_optimizer(new_polladdr, polladdr); - set_ctrl(new_polladdr, iffalse); - sfpt_clone->set_req(TypeFunc::Parms, new_polladdr); - } - // When this code runs, loop bodies have not yet been populated. - const bool body_populated = false; - register_control(sfpt_clone, outer_ilt, iffalse, body_populated); - set_idom(outer_le, sfpt_clone, dom_depth(sfpt_clone)); - } - lazy_replace(sfpt, sfpt->in(TypeFunc::Control)); - if (loop->_safepts != nullptr) { - loop->_safepts->yank(sfpt); - } - } - } - -#ifdef ASSERT - assert(l->is_valid_counted_loop(iv_bt), "counted loop shape is messed up"); - assert(l == loop->_head && l->phi() == phi && l->loopexit_or_null() == lex, "" ); -#endif -#ifndef PRODUCT - if (TraceLoopOpts) { - tty->print("Counted "); - loop->dump_head(); - } #endif - C->print_method(PHASE_AFTER_CLOOPS, 3, l); - - // Capture bounds of the loop in the induction variable Phi before - // subsequent transformation (iteration splitting) obscures the - // bounds - l->phi()->as_Phi()->set_type(l->phi()->Value(&_igvn)); - - if (strip_mine_loop) { - l->mark_strip_mined(); - l->verify_strip_mined(1); - outer_ilt->_head->as_Loop()->verify_strip_mined(1); - loop = outer_ilt; - } - -#ifndef PRODUCT - if (head->as_Loop()->is_loop_nest_inner_loop() && iv_bt == T_LONG) { - Atomic::inc(&_long_loop_counted_loops); - } -#endif - if (iv_bt == T_LONG && head->as_Loop()->is_loop_nest_outer_loop()) { - l->mark_loop_nest_outer_loop(); + loop = converter.convert(); + return true; } - return true; + return false; } bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { @@ -2850,18 +2125,6 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { } } -#ifdef ASSERT - // TODO: makes more sense to extract to a separate function? - // Stress and convert to long loop if StressLongCountedLoop flag is set - if (_iv_bt == T_INT && - !_head->as_Loop()->is_loop_nest_inner_loop() && - StressLongCountedLoop > 0 && - increment.trunc1 == nullptr && - _phase->convert_to_long_loop(exit_test.cmp, phi, _loop)) { - return false; - } -#endif - // TODO: saving only needed variables to context _limit = exit_test.limit; _stride_con = stride_con; @@ -2875,6 +2138,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { _cmp = exit_test.cmp; _cl_prob = exit_test.cl_prob; _sfpt = sfpt; + _trunc1 = increment.trunc1; #ifdef ASSERT _checked_for_counted_loop = true; @@ -3126,6 +2390,125 @@ IdealLoopTree* PhaseIdealLoop::CountedLoopConverter::convert() { return _loop; } +#ifdef ASSERT +bool PhaseIdealLoop::CountedLoopConverter::should_stress_long_counted_loop() const { + assert(_checked_for_counted_loop, "must check for counted loop before conversion"); + + return StressLongCountedLoop > 0 && + _iv_bt == T_INT && + !_head->as_Loop()->is_loop_nest_inner_loop() && + _trunc1 == nullptr; +} + +// Convert an int counted loop to a long counted to stress handling of long counted loops. Returns true upon success. +bool PhaseIdealLoop::CountedLoopConverter::stress_long_counted_loop() const { + assert(should_stress_long_counted_loop(), "stress condition not satisfied"); + + PhaseIterGVN* igvn = &_phase->_igvn; + Unique_Node_List iv_nodes; + Node_List old_new; + iv_nodes.push(_cmp); + bool failed = false; + + for (uint i = 0; i < iv_nodes.size() && !failed; i++) { + Node* n = iv_nodes.at(i); + switch(n->Opcode()) { + case Op_Phi: { + Node* clone = new PhiNode(n->in(0), TypeLong::LONG); + old_new.map(n->_idx, clone); + break; + } + case Op_CmpI: { + Node* clone = new CmpLNode(nullptr, nullptr); + old_new.map(n->_idx, clone); + break; + } + case Op_AddI: { + Node* clone = new AddLNode(nullptr, nullptr); + old_new.map(n->_idx, clone); + break; + } + case Op_CastII: { + failed = true; + break; + } + default: + DEBUG_ONLY(n->dump()); + fatal("unexpected"); + } + + for (uint j = 1; j < n->req(); j++) { + Node* in = n->in(j); + if (in == nullptr) { + continue; + } + if (_loop->is_member(_phase->get_loop(_phase->get_ctrl(in)))) { + iv_nodes.push(in); + } + } + } + + if (failed) { + for (uint i = 0; i < iv_nodes.size(); i++) { + Node* n = iv_nodes.at(i); + Node* clone = old_new[n->_idx]; + if (clone != nullptr) { + igvn->remove_dead_node(clone); + } + } + return false; + } + + for (uint i = 0; i < iv_nodes.size(); i++) { + Node* n = iv_nodes.at(i); + Node* clone = old_new[n->_idx]; + for (uint j = 1; j < n->req(); j++) { + Node* in = n->in(j); + if (in == nullptr) { + continue; + } + Node* in_clone = old_new[in->_idx]; + if (in_clone == nullptr) { + assert(igvn->type(in)->isa_int(), ""); + in_clone = new ConvI2LNode(in); + igvn->register_new_node_with_optimizer(in_clone); + _phase->set_subtree_ctrl(in_clone, false); + } + if (in_clone->in(0) == nullptr) { + in_clone->set_req(0, _phase->C->top()); + clone->set_req(j, in_clone); + in_clone->set_req(0, nullptr); + } else { + clone->set_req(j, in_clone); + } + } + igvn->register_new_node_with_optimizer(clone); + } + _phase->set_ctrl(old_new[_phi->_idx], _phi->in(0)); + + for (uint i = 0; i < iv_nodes.size(); i++) { + Node* n = iv_nodes.at(i); + Node* clone = old_new[n->_idx]; + _phase->set_subtree_ctrl(clone, false); + Node* m = n->Opcode() == Op_CmpI ? clone : nullptr; + for (DUIterator_Fast imax, j = n->fast_outs(imax); j < imax; j++) { + Node* u = n->fast_out(j); + if (iv_nodes.member(u)) { + continue; + } + if (m == nullptr) { + m = new ConvL2INode(clone); + igvn->register_new_node_with_optimizer(m); + _phase->set_subtree_ctrl(m, false); + } + igvn->rehash_node_delayed(u); + int nb = u->replace_edge(n, m, igvn); + --j, imax -= nb; + } + } + return true; +} +#endif // Check if there is a dominating loop limit check of the form 'init < limit' starting at the loop entry. // If there is one, then we do not need to create an additional Loop Limit Check Predicate. diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 3f1cbd3a02ec9..87247f9f97524 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1332,6 +1332,7 @@ class PhaseIdealLoop : public PhaseTransform { float _cl_prob; Node* _sfpt; jlong _final_correction; + Node* _trunc1; public: CountedLoopConverter(PhaseIdealLoop* phase, Node* head, IdealLoopTree* loop, const BasicType iv_bt) @@ -1346,6 +1347,11 @@ class PhaseIdealLoop : public PhaseTransform { bool is_counted_loop(); IdealLoopTree* convert(); + +#ifdef ASSERT + bool should_stress_long_counted_loop() const; + bool stress_long_counted_loop() const; +#endif }; Node* exact_limit( IdealLoopTree *loop ); From 8a5aba42747d4fd0f1338a86703040515ab3bdd6 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Fri, 4 Apr 2025 16:30:39 -0400 Subject: [PATCH 08/15] clean up code --- src/hotspot/share/opto/loopnode.cpp | 131 +--------------------------- src/hotspot/share/opto/loopnode.hpp | 6 +- 2 files changed, 5 insertions(+), 132 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index bde78efa124a5..0495151f928da 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -1465,34 +1465,19 @@ void PhaseIdealLoop::check_counted_loop_shape(IdealLoopTree* loop, Node* x, Basi Node* back_control = loop_exit_control(x, loop); assert(back_control != nullptr, "no back control"); -// BoolTest::mask mask = BoolTest::illegal; -// float cl_prob = 0; -// Node* incr = nullptr; -// Node* limit = nullptr; - -// Node* cmp = loop_exit_test(back_control, loop, incr, limit, mask, cl_prob); LoopExitTest exit_test = loop_exit_test(back_control, loop); assert(exit_test.cmp != nullptr && exit_test.cmp->Opcode() == Op_Cmp(bt), "no exit test"); -// const Node* phi_incr = incr; -// incr = loop_iv_incr(incr, x, loop); const LoopIVIncr iv_incr = loop_iv_incr(exit_test.incr, x, loop); -// phi_incr = iv_incr.phi_incr; -// incr = iv_incr.incr; assert(iv_incr.incr != nullptr && iv_incr.incr->Opcode() == Op_Add(bt), "no incr"); -// Node* xphi = nullptr; -// Node* stride = loop_iv_stride(iv_incr.incr, xphi); LoopIvStride stride = loop_iv_stride(iv_incr.incr); - assert(stride.stride != nullptr, "no stride"); PhiNode* phi = loop_iv_phi(stride.xphi, iv_incr.phi_incr, x); - assert(phi != nullptr && phi->in(LoopNode::LoopBackControl) == iv_incr.incr, "No phi"); jlong stride_con = stride.stride->get_integer_as_long(bt); - assert(condition_stride_ok(exit_test.mask, stride_con), "illegal condition"); assert(exit_test.mask != BoolTest::ne, "unexpected condition"); @@ -1504,117 +1489,6 @@ void PhaseIdealLoop::check_counted_loop_shape(IdealLoopTree* loop, Node* x, Basi } #endif -#ifdef ASSERT -// convert an int counted loop to a long counted to stress handling of -// long counted loops -bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, const Node* phi, const IdealLoopTree* loop) { - Unique_Node_List iv_nodes; - Node_List old_new; - iv_nodes.push(cmp); - bool failed = false; - - for (uint i = 0; i < iv_nodes.size() && !failed; i++) { - Node* n = iv_nodes.at(i); - switch(n->Opcode()) { - case Op_Phi: { - Node* clone = new PhiNode(n->in(0), TypeLong::LONG); - old_new.map(n->_idx, clone); - break; - } - case Op_CmpI: { - Node* clone = new CmpLNode(nullptr, nullptr); - old_new.map(n->_idx, clone); - break; - } - case Op_AddI: { - Node* clone = new AddLNode(nullptr, nullptr); - old_new.map(n->_idx, clone); - break; - } - case Op_CastII: { - failed = true; - break; - } - default: - DEBUG_ONLY(n->dump()); - fatal("unexpected"); - } - - for (uint i = 1; i < n->req(); i++) { - Node* in = n->in(i); - if (in == nullptr) { - continue; - } - if (loop->is_member(get_loop(get_ctrl(in)))) { - iv_nodes.push(in); - } - } - } - - if (failed) { - for (uint i = 0; i < iv_nodes.size(); i++) { - Node* n = iv_nodes.at(i); - Node* clone = old_new[n->_idx]; - if (clone != nullptr) { - _igvn.remove_dead_node(clone); - } - } - return false; - } - - for (uint i = 0; i < iv_nodes.size(); i++) { - Node* n = iv_nodes.at(i); - Node* clone = old_new[n->_idx]; - for (uint i = 1; i < n->req(); i++) { - Node* in = n->in(i); - if (in == nullptr) { - continue; - } - Node* in_clone = old_new[in->_idx]; - if (in_clone == nullptr) { - assert(_igvn.type(in)->isa_int(), ""); - in_clone = new ConvI2LNode(in); - _igvn.register_new_node_with_optimizer(in_clone); - set_subtree_ctrl(in_clone, false); - } - if (in_clone->in(0) == nullptr) { - in_clone->set_req(0, C->top()); - clone->set_req(i, in_clone); - in_clone->set_req(0, nullptr); - } else { - clone->set_req(i, in_clone); - } - } - _igvn.register_new_node_with_optimizer(clone); - } - set_ctrl(old_new[phi->_idx], phi->in(0)); - - for (uint i = 0; i < iv_nodes.size(); i++) { - Node* n = iv_nodes.at(i); - Node* clone = old_new[n->_idx]; - set_subtree_ctrl(clone, false); - Node* m = n->Opcode() == Op_CmpI ? clone : nullptr; - for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { - Node* u = n->fast_out(i); - if (iv_nodes.member(u)) { - continue; - } - if (m == nullptr) { - m = new ConvL2INode(clone); - _igvn.register_new_node_with_optimizer(m); - set_subtree_ctrl(m, false); - } - _igvn.rehash_node_delayed(u); - int nb = u->replace_edge(n, m, &_igvn); - --i, imax -= nb; - } - } - return true; -} -#endif - - - //------------------------------try_convert_to_counted_loop-------------------------------- bool PhaseIdealLoop::try_convert_to_counted_loop(Node* head, IdealLoopTree*& loop, const BasicType iv_bt) { CountedLoopConverter converter(this, head, loop, iv_bt); @@ -2125,7 +1999,8 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { } } - // TODO: saving only needed variables to context + // Variables needed by convert() to do the actual conversion. We set these fields iff a counted loop is confirmed. + // This way the is_counted_loop() can run multiple times with possibly different configurations. _limit = exit_test.limit; _stride_con = stride_con; _final_correction = final_correction; @@ -2270,7 +2145,7 @@ IdealLoopTree* PhaseIdealLoop::CountedLoopConverter::convert() { Node* ift2 = igvn->register_new_node_with_optimizer(new IfTrueNode(le)); Node* iff2 = igvn->register_new_node_with_optimizer(new IfFalseNode(le)); - _loop->_tail = back_control = ift2; // XXX: mutation + _loop->_tail = back_control = ift2; _phase->set_loop(ift2, _loop); _phase->set_loop(iff2, _phase->get_loop(iffalse)); diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 87247f9f97524..6c6d93088676a 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1290,14 +1290,11 @@ class PhaseIdealLoop : public PhaseTransform { static LoopIvStride loop_iv_stride(const Node* incr); static PhiNode* loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x); - bool try_convert_to_counted_loop_old(Node* head, IdealLoopTree*& loop, const BasicType iv_bt); bool try_convert_to_counted_loop(Node* head, IdealLoopTree*&loop, const BasicType iv_bt); Node* loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv, Node* outer_phi, Node* inner_head, BasicType bt); bool create_loop_nest(IdealLoopTree* loop, Node_List &old_new); -#ifdef ASSERT - bool convert_to_long_loop(Node* cmp, const Node* phi, const IdealLoopTree* loop); -#endif + void add_parse_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop, SafePointNode* sfpt); SafePointNode* find_safepoint(Node* back_control, const Node* x, const IdealLoopTree* loop); IdealLoopTree* insert_outer_loop(IdealLoopTree* loop, LoopNode* outer_l, Node* outer_ift); @@ -1340,6 +1337,7 @@ class PhaseIdealLoop : public PhaseTransform { _head(head), _loop(loop), _iv_bt(iv_bt) { + assert(phase != nullptr, ""); assert(head != nullptr, ""); assert(loop != nullptr, ""); assert(iv_bt == T_INT || iv_bt == T_LONG, ""); From cef7b08ce74c17474cd2167a743697e3bd9edacb Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Mon, 7 Apr 2025 11:36:59 -0400 Subject: [PATCH 09/15] rearrange code for a more sensible diff --- src/hotspot/share/opto/loopnode.cpp | 242 ++++++++++++++-------------- src/hotspot/share/opto/loopopts.cpp | 17 -- 2 files changed, 121 insertions(+), 138 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 0495151f928da..b4558fd5ae91e 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -1489,7 +1489,127 @@ void PhaseIdealLoop::check_counted_loop_shape(IdealLoopTree* loop, Node* x, Basi } #endif -//------------------------------try_convert_to_counted_loop-------------------------------- +//------------------------------CountedLoopConverter-------------------------------- +#ifdef ASSERT +bool PhaseIdealLoop::CountedLoopConverter::should_stress_long_counted_loop() const { + assert(_checked_for_counted_loop, "must check for counted loop before conversion"); + + return StressLongCountedLoop > 0 && + _iv_bt == T_INT && + !_head->as_Loop()->is_loop_nest_inner_loop() && + _trunc1 == nullptr; +} + +// Convert an int counted loop to a long counted to stress handling of long counted loops. Returns true upon success. +bool PhaseIdealLoop::CountedLoopConverter::stress_long_counted_loop() const { + assert(should_stress_long_counted_loop(), "stress condition not satisfied"); + + PhaseIterGVN* igvn = &_phase->_igvn; + Unique_Node_List iv_nodes; + Node_List old_new; + iv_nodes.push(_cmp); + bool failed = false; + + for (uint i = 0; i < iv_nodes.size() && !failed; i++) { + Node* n = iv_nodes.at(i); + switch(n->Opcode()) { + case Op_Phi: { + Node* clone = new PhiNode(n->in(0), TypeLong::LONG); + old_new.map(n->_idx, clone); + break; + } + case Op_CmpI: { + Node* clone = new CmpLNode(nullptr, nullptr); + old_new.map(n->_idx, clone); + break; + } + case Op_AddI: { + Node* clone = new AddLNode(nullptr, nullptr); + old_new.map(n->_idx, clone); + break; + } + case Op_CastII: { + failed = true; + break; + } + default: + DEBUG_ONLY(n->dump()); + fatal("unexpected"); + } + + for (uint j = 1; j < n->req(); j++) { + Node* in = n->in(j); + if (in == nullptr) { + continue; + } + if (_loop->is_member(_phase->get_loop(_phase->get_ctrl(in)))) { + iv_nodes.push(in); + } + } + } + + if (failed) { + for (uint i = 0; i < iv_nodes.size(); i++) { + Node* n = iv_nodes.at(i); + Node* clone = old_new[n->_idx]; + if (clone != nullptr) { + igvn->remove_dead_node(clone); + } + } + return false; + } + + for (uint i = 0; i < iv_nodes.size(); i++) { + Node* n = iv_nodes.at(i); + Node* clone = old_new[n->_idx]; + for (uint j = 1; j < n->req(); j++) { + Node* in = n->in(j); + if (in == nullptr) { + continue; + } + Node* in_clone = old_new[in->_idx]; + if (in_clone == nullptr) { + assert(igvn->type(in)->isa_int(), ""); + in_clone = new ConvI2LNode(in); + igvn->register_new_node_with_optimizer(in_clone); + _phase->set_subtree_ctrl(in_clone, false); + } + if (in_clone->in(0) == nullptr) { + in_clone->set_req(0, _phase->C->top()); + clone->set_req(j, in_clone); + in_clone->set_req(0, nullptr); + } else { + clone->set_req(j, in_clone); + } + } + igvn->register_new_node_with_optimizer(clone); + } + _phase->set_ctrl(old_new[_phi->_idx], _phi->in(0)); + + for (uint i = 0; i < iv_nodes.size(); i++) { + Node* n = iv_nodes.at(i); + Node* clone = old_new[n->_idx]; + _phase->set_subtree_ctrl(clone, false); + Node* m = n->Opcode() == Op_CmpI ? clone : nullptr; + for (DUIterator_Fast imax, j = n->fast_outs(imax); j < imax; j++) { + Node* u = n->fast_out(j); + if (iv_nodes.member(u)) { + continue; + } + if (m == nullptr) { + m = new ConvL2INode(clone); + igvn->register_new_node_with_optimizer(m); + _phase->set_subtree_ctrl(m, false); + } + igvn->rehash_node_delayed(u); + int nb = u->replace_edge(n, m, igvn); + --j, imax -= nb; + } + } + return true; +} +#endif + bool PhaseIdealLoop::try_convert_to_counted_loop(Node* head, IdealLoopTree*& loop, const BasicType iv_bt) { CountedLoopConverter converter(this, head, loop, iv_bt); if (converter.is_counted_loop()) { @@ -2265,126 +2385,6 @@ IdealLoopTree* PhaseIdealLoop::CountedLoopConverter::convert() { return _loop; } -#ifdef ASSERT -bool PhaseIdealLoop::CountedLoopConverter::should_stress_long_counted_loop() const { - assert(_checked_for_counted_loop, "must check for counted loop before conversion"); - - return StressLongCountedLoop > 0 && - _iv_bt == T_INT && - !_head->as_Loop()->is_loop_nest_inner_loop() && - _trunc1 == nullptr; -} - -// Convert an int counted loop to a long counted to stress handling of long counted loops. Returns true upon success. -bool PhaseIdealLoop::CountedLoopConverter::stress_long_counted_loop() const { - assert(should_stress_long_counted_loop(), "stress condition not satisfied"); - - PhaseIterGVN* igvn = &_phase->_igvn; - Unique_Node_List iv_nodes; - Node_List old_new; - iv_nodes.push(_cmp); - bool failed = false; - - for (uint i = 0; i < iv_nodes.size() && !failed; i++) { - Node* n = iv_nodes.at(i); - switch(n->Opcode()) { - case Op_Phi: { - Node* clone = new PhiNode(n->in(0), TypeLong::LONG); - old_new.map(n->_idx, clone); - break; - } - case Op_CmpI: { - Node* clone = new CmpLNode(nullptr, nullptr); - old_new.map(n->_idx, clone); - break; - } - case Op_AddI: { - Node* clone = new AddLNode(nullptr, nullptr); - old_new.map(n->_idx, clone); - break; - } - case Op_CastII: { - failed = true; - break; - } - default: - DEBUG_ONLY(n->dump()); - fatal("unexpected"); - } - - for (uint j = 1; j < n->req(); j++) { - Node* in = n->in(j); - if (in == nullptr) { - continue; - } - if (_loop->is_member(_phase->get_loop(_phase->get_ctrl(in)))) { - iv_nodes.push(in); - } - } - } - - if (failed) { - for (uint i = 0; i < iv_nodes.size(); i++) { - Node* n = iv_nodes.at(i); - Node* clone = old_new[n->_idx]; - if (clone != nullptr) { - igvn->remove_dead_node(clone); - } - } - return false; - } - - for (uint i = 0; i < iv_nodes.size(); i++) { - Node* n = iv_nodes.at(i); - Node* clone = old_new[n->_idx]; - for (uint j = 1; j < n->req(); j++) { - Node* in = n->in(j); - if (in == nullptr) { - continue; - } - Node* in_clone = old_new[in->_idx]; - if (in_clone == nullptr) { - assert(igvn->type(in)->isa_int(), ""); - in_clone = new ConvI2LNode(in); - igvn->register_new_node_with_optimizer(in_clone); - _phase->set_subtree_ctrl(in_clone, false); - } - if (in_clone->in(0) == nullptr) { - in_clone->set_req(0, _phase->C->top()); - clone->set_req(j, in_clone); - in_clone->set_req(0, nullptr); - } else { - clone->set_req(j, in_clone); - } - } - igvn->register_new_node_with_optimizer(clone); - } - _phase->set_ctrl(old_new[_phi->_idx], _phi->in(0)); - - for (uint i = 0; i < iv_nodes.size(); i++) { - Node* n = iv_nodes.at(i); - Node* clone = old_new[n->_idx]; - _phase->set_subtree_ctrl(clone, false); - Node* m = n->Opcode() == Op_CmpI ? clone : nullptr; - for (DUIterator_Fast imax, j = n->fast_outs(imax); j < imax; j++) { - Node* u = n->fast_out(j); - if (iv_nodes.member(u)) { - continue; - } - if (m == nullptr) { - m = new ConvL2INode(clone); - igvn->register_new_node_with_optimizer(m); - _phase->set_subtree_ctrl(m, false); - } - igvn->rehash_node_delayed(u); - int nb = u->replace_edge(n, m, igvn); - --j, imax -= nb; - } - } - return true; -} -#endif - // Check if there is a dominating loop limit check of the form 'init < limit' starting at the loop entry. // If there is one, then we do not need to create an additional Loop Limit Check Predicate. bool PhaseIdealLoop::has_dominating_loop_limit_check(Node* init_trip, Node* limit, const jlong stride_con, diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index c1f44c5631cc6..defb4548d84f3 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -2848,8 +2848,6 @@ int PhaseIdealLoop::stride_of_possible_iv(Node* iff) { Node* phi = cmp1; for (uint i = 1; i < phi->req(); i++) { Node* in = phi->in(i); -// Node* add = CountedLoopNode::match_incr_with_optional_truncation(in, -// &trunc1, &trunc2, &ttype, T_INT); CountedLoopNode::TruncatedIncrement increment = CountedLoopNode::match_incr_with_optional_truncation(in, T_INT); if (increment.incr != nullptr && increment.incr->in(1) == phi) { add2 = increment.incr->in(2); @@ -2859,8 +2857,6 @@ int PhaseIdealLoop::stride_of_possible_iv(Node* iff) { } else { // (If (Bool (CmpX addtrunc:(Optional-trunc((AddI (Phi ...addtrunc...) add2)) ))) Node* addtrunc = cmp1; -// Node* add = CountedLoopNode::match_incr_with_optional_truncation(addtrunc, -// &trunc1, &trunc2, &ttype, T_INT); CountedLoopNode::TruncatedIncrement increment = CountedLoopNode::match_incr_with_optional_truncation(addtrunc, T_INT); if (increment.incr != nullptr && increment.incr->in(1)->is_Phi()) { Node* phi = increment.incr->in(1); @@ -4270,11 +4266,6 @@ bool PhaseIdealLoop::duplicate_loop_backedge(IdealLoopTree *loop, Node_List &old return false; } -// BoolTest::mask bt = BoolTest::illegal; -// float cl_prob = 0; -// Node* incr = nullptr; -// Node* limit = nullptr; -// Node* cmp = loop_exit_test(back_control, loop, incr, limit, bt, cl_prob); LoopExitTest loop_exit = loop_exit_test(back_control, loop); if (loop_exit.cmp == nullptr || loop_exit.cmp->Opcode() != Op_CmpI) { return false; @@ -4294,19 +4285,11 @@ bool PhaseIdealLoop::duplicate_loop_backedge(IdealLoopTree *loop, Node_List &old // if the extra phi is removed inner = 0; for (uint i = 1; i < loop_exit.incr->req(); ++i) { -// Node* in = incr->in(i); -// Node* trunc1 = nullptr; -// Node* trunc2 = nullptr; -// const TypeInteger* iv_trunc_t = nullptr; -// Node* orig_in = in; CountedLoopNode::TruncatedIncrement increment = CountedLoopNode::match_incr_with_optional_truncation(loop_exit.incr->in(i), T_INT); -// if (!(in = CountedLoopNode::match_incr_with_optional_truncation(in, &trunc1, &trunc2, &iv_trunc_t, T_INT))) { if (increment.incr == nullptr) { continue; } assert(increment.incr->Opcode() == Op_AddI, "wrong increment code"); -// Node* xphi = nullptr; -// Node* stride = loop_iv_stride(increment.incr, xphi); LoopIvStride stride = loop_iv_stride(increment.incr); if (stride.stride == nullptr) { From c4e2dffa6814b71c1798bc2f1bcf4aa0f7e18114 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Mon, 7 Apr 2025 13:39:43 -0400 Subject: [PATCH 10/15] improve formatting, naming, comments --- src/hotspot/share/opto/loopnode.cpp | 54 +++++++++++++---------------- src/hotspot/share/opto/loopnode.hpp | 2 +- src/hotspot/share/opto/loopopts.cpp | 14 ++++---- 3 files changed, 33 insertions(+), 37 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index b4558fd5ae91e..f79d22900b4b1 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -404,10 +404,7 @@ Node* PhaseIdealLoop::loop_exit_control(const Node* x, const IdealLoopTree* loop return iftrue; } -//Node* PhaseIdealLoop::loop_exit_test(const Node* back_control, const IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob) { PhaseIdealLoop::LoopExitTest PhaseIdealLoop::loop_exit_test(const Node* back_control, const IdealLoopTree* loop) { - - const Node* iftrue = back_control; uint iftrue_op = iftrue->Opcode(); Node* iff = iftrue->in(0); @@ -446,9 +443,9 @@ PhaseIdealLoop::LoopExitTest PhaseIdealLoop::loop_exit_test(const Node* back_con } PhaseIdealLoop::LoopIVIncr PhaseIdealLoop::loop_iv_incr(Node* incr, const Node* x, const IdealLoopTree* loop) { - if (incr->is_Phi() && incr->as_Phi()->region() == x && incr->req() == 3) { // Simple trip counter expression + if (incr->is_Phi() && incr->as_Phi()->region() == x && incr->req() == 3) { // Requires simple trip counter expression Node* phi_incr = incr; - Node* back_control = phi_incr->in(LoopNode::LoopBackControl); // Assume old_incr is on backedge of Phi + Node* back_control = phi_incr->in(LoopNode::LoopBackControl); // Assume incr is on backedge of Phi if (loop->_phase->is_member(loop, loop->_phase->get_ctrl(back_control))) { // Trip counter must be loop-variant return {back_control, phi_incr}; } @@ -463,7 +460,7 @@ PhaseIdealLoop::LoopIvStride PhaseIdealLoop::loop_iv_stride(const Node* incr) { Node *stride = incr->in(2); if (!stride->is_Con()) { // Oops, swap these if (!xphi->is_Con()) { // Is the other guy a constant? - return {}; // Nope, unknown stride, bail out + return {}; // Nope, unknown stride, bail out } swap(xphi, stride); // 'incr' is commutative, so ok to swap @@ -1492,7 +1489,7 @@ void PhaseIdealLoop::check_counted_loop_shape(IdealLoopTree* loop, Node* x, Basi //------------------------------CountedLoopConverter-------------------------------- #ifdef ASSERT bool PhaseIdealLoop::CountedLoopConverter::should_stress_long_counted_loop() const { - assert(_checked_for_counted_loop, "must check for counted loop before conversion"); + assert(_checked_for_counted_loop, "must check for counted loop before stressing"); return StressLongCountedLoop > 0 && _iv_bt == T_INT && @@ -1628,7 +1625,7 @@ bool PhaseIdealLoop::try_convert_to_counted_loop(Node* head, IdealLoopTree*& loo } bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { - PhaseGVN* igvn = &_phase->_igvn; + PhaseIterGVN* igvn = &_phase->_igvn; Node* back_control = _phase->loop_exit_control(_head, _loop); if (back_control == nullptr) { @@ -2083,17 +2080,17 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { _insert_init_trip_limit_check = true; } - BoolTest::mask bt = exit_test.mask; - if (bt == BoolTest::ne) { + BoolTest::mask mask = exit_test.mask; + if (mask == BoolTest::ne) { // Now we need to canonicalize the loop condition if it is 'ne'. assert(stride_con == 1 || stride_con == -1, "simple increment only - checked before"); if (stride_con > 0) { // 'ne' can be replaced with 'lt' only when init < limit. This is ensured by the inserted predicate above. - bt = BoolTest::lt; + mask = BoolTest::lt; } else { assert(stride_con < 0, "must be"); // 'ne' can be replaced with 'gt' only when init > limit. This is ensured by the inserted predicate above. - bt = BoolTest::gt; + mask = BoolTest::gt; } } @@ -2128,7 +2125,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { _phi_incr = iv_incr.phi_incr; _stride = stride.stride; _includes_limit = includes_limit; - _bt = bt; + _mask = mask; _increment = increment.incr; _cmp = exit_test.cmp; _cl_prob = exit_test.cl_prob; @@ -2192,16 +2189,17 @@ IdealLoopTree* PhaseIdealLoop::CountedLoopConverter::convert() { adjusted_limit = igvn->transform(AddNode::make(_limit, _stride, _iv_bt)); } + BoolTest::mask mask = _mask; if (_includes_limit) { // The limit check guaranties that 'limit <= (max_jint - stride)' so // we can convert 'i <= limit' to 'i < limit+1' since stride != 0. // Node* one = (_stride_con > 0) ? igvn->integercon(1, _iv_bt) : igvn->integercon(-1, _iv_bt); adjusted_limit = igvn->transform(AddNode::make(adjusted_limit, one, _iv_bt)); - if (_bt == BoolTest::le) - _bt = BoolTest::lt; - else if (_bt == BoolTest::ge) - _bt = BoolTest::gt; + if (mask == BoolTest::le) + mask = BoolTest::lt; + else if (mask == BoolTest::ge) + mask = BoolTest::gt; else ShouldNotReachHere(); } @@ -2209,13 +2207,13 @@ IdealLoopTree* PhaseIdealLoop::CountedLoopConverter::convert() { // Build a canonical trip test. // Clone code, as old values may be in use. - Node* incr_clone = _increment->clone(); - incr_clone->set_req(1, _phi); - incr_clone->set_req(2, _stride); - incr_clone = igvn->register_new_node_with_optimizer(incr_clone); - _phase->set_early_ctrl(incr_clone, false); + Node* incr = _increment->clone(); + incr->set_req(1, _phi); + incr->set_req(2, _stride); + incr = igvn->register_new_node_with_optimizer(incr); + _phase->set_early_ctrl(incr, false); igvn->rehash_node_delayed(_phi); - _phi->set_req_X(LoopNode::LoopBackControl, incr_clone, igvn); + _phi->set_req_X(LoopNode::LoopBackControl, incr, igvn); // If phi type is more restrictive than Int, raise to // Int to prevent (almost) infinite recursion in igvn @@ -2234,14 +2232,14 @@ IdealLoopTree* PhaseIdealLoop::CountedLoopConverter::convert() { const uint iftrue_op = iftrue->Opcode(); Node* iff = iftrue->in(0); Node* cmp = _cmp->clone(); - BoolNode* test = iff->in(1)->clone()->as_Bool(); - cmp->set_req(1, incr_clone); + cmp->set_req(1, incr); cmp->set_req(2, adjusted_limit); cmp = igvn->register_new_node_with_optimizer(cmp); _phase->set_ctrl(cmp, iff->in(0)); - const_cast(&test->_test)->_test = _bt; // Yes, it's const, but it's newly cloned node so it should be fine. + BoolNode* test = iff->in(1)->clone()->as_Bool(); + const_cast(&test->_test)->_test = mask; // Yes, it's a const, but it's a newly cloned node so we should be fine. test->set_req(1, cmp); igvn->register_new_node_with_optimizer(test); _phase->set_ctrl(test, iff->in(0)); @@ -2296,9 +2294,7 @@ IdealLoopTree* PhaseIdealLoop::CountedLoopConverter::convert() { _phase->is_deleteable_safept(_sfpt); IdealLoopTree* outer_ilt = nullptr; if (strip_mine_loop) { - outer_ilt = _phase->create_outer_strip_mined_loop(init_control, _loop, - _cl_prob, le->_fcnt, entry_control, - iffalse); + outer_ilt = _phase->create_outer_strip_mined_loop(init_control, _loop, _cl_prob, le->_fcnt, entry_control, iffalse); } // Now setup a new CountedLoopNode to replace the existing LoopNode diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 6c6d93088676a..2ce0317a7bb08 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1323,7 +1323,7 @@ class PhaseIdealLoop : public PhaseTransform { Node* _phi_incr; Node* _stride; bool _includes_limit; - BoolTest::mask _bt; + BoolTest::mask _mask; Node* _increment; Node* _cmp; float _cl_prob; diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index defb4548d84f3..7823c9b72ef6b 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -2848,21 +2848,21 @@ int PhaseIdealLoop::stride_of_possible_iv(Node* iff) { Node* phi = cmp1; for (uint i = 1; i < phi->req(); i++) { Node* in = phi->in(i); - CountedLoopNode::TruncatedIncrement increment = CountedLoopNode::match_incr_with_optional_truncation(in, T_INT); - if (increment.incr != nullptr && increment.incr->in(1) == phi) { - add2 = increment.incr->in(2); + CountedLoopNode::TruncatedIncrement add = CountedLoopNode::match_incr_with_optional_truncation(in, T_INT); + if (add.incr != nullptr && add.incr->in(1) == phi) { + add2 = add.incr->in(2); break; } } } else { // (If (Bool (CmpX addtrunc:(Optional-trunc((AddI (Phi ...addtrunc...) add2)) ))) Node* addtrunc = cmp1; - CountedLoopNode::TruncatedIncrement increment = CountedLoopNode::match_incr_with_optional_truncation(addtrunc, T_INT); - if (increment.incr != nullptr && increment.incr->in(1)->is_Phi()) { - Node* phi = increment.incr->in(1); + CountedLoopNode::TruncatedIncrement add = CountedLoopNode::match_incr_with_optional_truncation(addtrunc, T_INT); + if (add.incr != nullptr && add.incr->in(1)->is_Phi()) { + Node* phi = add.incr->in(1); for (uint i = 1; i < phi->req(); i++) { if (phi->in(i) == addtrunc) { - add2 = increment.incr->in(2); + add2 = add.incr->in(2); break; } } From fd6071761bdc47ab5695559dffd1e1dd6038d9f7 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Mon, 7 Apr 2025 14:45:38 -0400 Subject: [PATCH 11/15] improve formatting, naming, comments --- src/hotspot/share/opto/loopnode.cpp | 8 ++++---- src/hotspot/share/opto/loopnode.hpp | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 502de9989e49b..ab0e635066a69 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -2003,6 +2003,8 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { const Predicates predicates(init_control); const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); + const ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); + Node* parse_predicate_entry = loop_limit_check_parse_predicate->in(0); if (sov < 0) { return false; // Bailout: integer overflow is certain. @@ -2026,8 +2028,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { return false; } - ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); - if (!_phase->is_dominator(_phase->get_ctrl(exit_test.limit), loop_limit_check_parse_predicate->in(0))) { + if (!_phase->is_dominator(_phase->get_ctrl(exit_test.limit), parse_predicate_entry)) { return false; } @@ -2070,8 +2071,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { return false; } - ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); - Node* parse_predicate_entry = loop_limit_check_parse_predicate->in(0); + if (!_phase->is_dominator(_phase->get_ctrl(exit_test.limit), parse_predicate_entry) || !_phase->is_dominator(_phase->get_ctrl(init_trip), parse_predicate_entry)) { return false; diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 225677d1b31da..ab1dd49adbe71 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1271,6 +1271,7 @@ class PhaseIdealLoop : public PhaseTransform { virtual Node* transform(Node* n) { return nullptr; } Node* loop_exit_control(const Node* x, const IdealLoopTree* loop) const; + struct LoopExitTest { Node* cmp = nullptr; Node* incr = nullptr; @@ -1279,16 +1280,19 @@ class PhaseIdealLoop : public PhaseTransform { const float cl_prob = 0.0f; }; LoopExitTest loop_exit_test(const Node* back_control, const IdealLoopTree* loop); + struct LoopIVIncr { Node* incr = nullptr; Node* phi_incr = nullptr; }; LoopIVIncr loop_iv_incr(Node* old_incr, const Node* x, const IdealLoopTree* loop); + struct LoopIvStride { Node* stride = nullptr; Node* xphi = nullptr; }; static LoopIvStride loop_iv_stride(const Node* incr); + static PhiNode* loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x); bool try_convert_to_counted_loop(Node* head, IdealLoopTree*&loop, const BasicType iv_bt); @@ -1303,14 +1307,12 @@ class PhaseIdealLoop : public PhaseTransform { IdealLoopTree* loop, float cl_prob, float le_fcnt, Node*& entry_control, Node*& iffalse); - // FIXME: move to somewhere more sensible class CountedLoopConverter { PhaseIdealLoop* _phase; Node* _head; IdealLoopTree* _loop; const BasicType _iv_bt; - // TODO: better naming for these flags? bool _insert_stride_overflow_limit_check = false; bool _insert_init_trip_limit_check = false; @@ -1318,6 +1320,7 @@ class PhaseIdealLoop : public PhaseTransform { bool _checked_for_counted_loop = false; #endif + // To be assigned once a counted loop is confirmed. Node* _limit; jlong _stride_con; Node* _phi; From fff8aee82b71f224618a477b788064275dfde2c8 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Mon, 7 Apr 2025 15:04:33 -0400 Subject: [PATCH 12/15] Revert "improve formatting, naming, comments" This reverts commit fd6071761bdc47ab5695559dffd1e1dd6038d9f7. --- src/hotspot/share/opto/loopnode.cpp | 8 ++++---- src/hotspot/share/opto/loopnode.hpp | 7 ++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index ab0e635066a69..502de9989e49b 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -2003,8 +2003,6 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { const Predicates predicates(init_control); const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); - const ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); - Node* parse_predicate_entry = loop_limit_check_parse_predicate->in(0); if (sov < 0) { return false; // Bailout: integer overflow is certain. @@ -2028,7 +2026,8 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { return false; } - if (!_phase->is_dominator(_phase->get_ctrl(exit_test.limit), parse_predicate_entry)) { + ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); + if (!_phase->is_dominator(_phase->get_ctrl(exit_test.limit), loop_limit_check_parse_predicate->in(0))) { return false; } @@ -2071,7 +2070,8 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { return false; } - + ParsePredicateNode* loop_limit_check_parse_predicate = loop_limit_check_predicate_block->parse_predicate(); + Node* parse_predicate_entry = loop_limit_check_parse_predicate->in(0); if (!_phase->is_dominator(_phase->get_ctrl(exit_test.limit), parse_predicate_entry) || !_phase->is_dominator(_phase->get_ctrl(init_trip), parse_predicate_entry)) { return false; diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index ab1dd49adbe71..225677d1b31da 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1271,7 +1271,6 @@ class PhaseIdealLoop : public PhaseTransform { virtual Node* transform(Node* n) { return nullptr; } Node* loop_exit_control(const Node* x, const IdealLoopTree* loop) const; - struct LoopExitTest { Node* cmp = nullptr; Node* incr = nullptr; @@ -1280,19 +1279,16 @@ class PhaseIdealLoop : public PhaseTransform { const float cl_prob = 0.0f; }; LoopExitTest loop_exit_test(const Node* back_control, const IdealLoopTree* loop); - struct LoopIVIncr { Node* incr = nullptr; Node* phi_incr = nullptr; }; LoopIVIncr loop_iv_incr(Node* old_incr, const Node* x, const IdealLoopTree* loop); - struct LoopIvStride { Node* stride = nullptr; Node* xphi = nullptr; }; static LoopIvStride loop_iv_stride(const Node* incr); - static PhiNode* loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x); bool try_convert_to_counted_loop(Node* head, IdealLoopTree*&loop, const BasicType iv_bt); @@ -1307,12 +1303,14 @@ class PhaseIdealLoop : public PhaseTransform { IdealLoopTree* loop, float cl_prob, float le_fcnt, Node*& entry_control, Node*& iffalse); + // FIXME: move to somewhere more sensible class CountedLoopConverter { PhaseIdealLoop* _phase; Node* _head; IdealLoopTree* _loop; const BasicType _iv_bt; + // TODO: better naming for these flags? bool _insert_stride_overflow_limit_check = false; bool _insert_init_trip_limit_check = false; @@ -1320,7 +1318,6 @@ class PhaseIdealLoop : public PhaseTransform { bool _checked_for_counted_loop = false; #endif - // To be assigned once a counted loop is confirmed. Node* _limit; jlong _stride_con; Node* _phi; From 5c1eb7970d40cee6eb4ab453be2062ea2bd3ae2c Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Mon, 7 Apr 2025 15:34:23 -0400 Subject: [PATCH 13/15] remove TODOs --- src/hotspot/share/opto/loopnode.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 225677d1b31da..ab1dd49adbe71 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1271,6 +1271,7 @@ class PhaseIdealLoop : public PhaseTransform { virtual Node* transform(Node* n) { return nullptr; } Node* loop_exit_control(const Node* x, const IdealLoopTree* loop) const; + struct LoopExitTest { Node* cmp = nullptr; Node* incr = nullptr; @@ -1279,16 +1280,19 @@ class PhaseIdealLoop : public PhaseTransform { const float cl_prob = 0.0f; }; LoopExitTest loop_exit_test(const Node* back_control, const IdealLoopTree* loop); + struct LoopIVIncr { Node* incr = nullptr; Node* phi_incr = nullptr; }; LoopIVIncr loop_iv_incr(Node* old_incr, const Node* x, const IdealLoopTree* loop); + struct LoopIvStride { Node* stride = nullptr; Node* xphi = nullptr; }; static LoopIvStride loop_iv_stride(const Node* incr); + static PhiNode* loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x); bool try_convert_to_counted_loop(Node* head, IdealLoopTree*&loop, const BasicType iv_bt); @@ -1303,14 +1307,12 @@ class PhaseIdealLoop : public PhaseTransform { IdealLoopTree* loop, float cl_prob, float le_fcnt, Node*& entry_control, Node*& iffalse); - // FIXME: move to somewhere more sensible class CountedLoopConverter { PhaseIdealLoop* _phase; Node* _head; IdealLoopTree* _loop; const BasicType _iv_bt; - // TODO: better naming for these flags? bool _insert_stride_overflow_limit_check = false; bool _insert_init_trip_limit_check = false; @@ -1318,6 +1320,7 @@ class PhaseIdealLoop : public PhaseTransform { bool _checked_for_counted_loop = false; #endif + // To be assigned once a counted loop is confirmed. Node* _limit; jlong _stride_con; Node* _phi; From 3db745d23ae4a32be094f3c6db4c12f2492909ed Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Wed, 9 Apr 2025 11:47:58 -0400 Subject: [PATCH 14/15] line break --- src/hotspot/share/opto/loopnode.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 502de9989e49b..790a0706316ca 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -2010,6 +2010,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { // If sov == 0, limit's type always satisfies the condition, for // example, when it is an array length. + if (sov > 0) { // (1) Loop Limit Check Predicate is required because we could not statically prove that // limit + final_correction = adjusted_limit - 1 + stride <= max_int From b72e7714786ab783395f3c5926ac30d7485c68e5 Mon Sep 17 00:00:00 2001 From: Kangcheng Xu Date: Thu, 10 Apr 2025 11:24:19 -0400 Subject: [PATCH 15/15] reviewer suggested changes --- src/hotspot/share/opto/loopnode.cpp | 41 ++++++++++++++--------------- src/hotspot/share/opto/loopnode.hpp | 15 +++++------ 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 790a0706316ca..04760b3d29a8c 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -310,7 +310,7 @@ IdealLoopTree* PhaseIdealLoop::insert_outer_loop(IdealLoopTree* loop, LoopNode* // clone the outer loop as well as the inner, unrolling needs to only // clone the inner loop etc. No optimizations need to change the outer // strip mined loop as it is only a skeleton. -IdealLoopTree* PhaseIdealLoop::create_outer_strip_mined_loop(Node *init_control, +IdealLoopTree* PhaseIdealLoop::create_outer_strip_mined_loop(Node* init_control, IdealLoopTree* loop, float cl_prob, float le_fcnt, Node*& entry_control, Node*& iffalse) { Node* outer_test = intcon(0); @@ -364,14 +364,14 @@ void PhaseIdealLoop::insert_loop_limit_check_predicate(const ParsePredicateSucce #endif } -Node* PhaseIdealLoop::loop_exit_control(const Node* x, const IdealLoopTree* loop) const { +Node* PhaseIdealLoop::loop_exit_control(const Node* head, const IdealLoopTree* loop) const { // Counted loop head must be a good RegionNode with only 3 not null // control input edges: Self, Entry, LoopBack. - if (x->in(LoopNode::Self) == nullptr || x->req() != 3 || loop->_irreducible) { + if (head->in(LoopNode::Self) == nullptr || head->req() != 3 || loop->_irreducible) { return nullptr; } - Node *init_control = x->in(LoopNode::EntryControl); - Node *back_control = x->in(LoopNode::LoopBackControl); + Node *init_control = head->in(LoopNode::EntryControl); + Node *back_control = head->in(LoopNode::LoopBackControl); if (init_control == nullptr || back_control == nullptr) { // Partially dead return nullptr; } @@ -442,8 +442,8 @@ PhaseIdealLoop::LoopExitTest PhaseIdealLoop::loop_exit_test(const Node* back_con return {cmp, incr, limit, bt, cl_prob}; } -PhaseIdealLoop::LoopIVIncr PhaseIdealLoop::loop_iv_incr(Node* incr, const Node* x, const IdealLoopTree* loop) { - if (incr->is_Phi() && incr->as_Phi()->region() == x && incr->req() == 3) { // Requires simple trip counter expression +PhaseIdealLoop::LoopIVIncr PhaseIdealLoop::loop_iv_incr(Node* incr, const Node* head, const IdealLoopTree* loop) { + if (incr->is_Phi() && incr->as_Phi()->region() == head && incr->req() == 3) { // Requires simple trip counter expression Node* phi_incr = incr; Node* back_control = phi_incr->in(LoopNode::LoopBackControl); // Assume incr is on backedge of Phi if (loop->_phase->is_member(loop, loop->_phase->get_ctrl(back_control))) { // Trip counter must be loop-variant @@ -468,7 +468,7 @@ PhaseIdealLoop::LoopIvStride PhaseIdealLoop::loop_iv_stride(const Node* incr) { return {stride, xphi}; } -PhiNode* PhaseIdealLoop::loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x) { +PhiNode* PhaseIdealLoop::loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* head) { if (!xphi->is_Phi()) { return nullptr; // Too much math on the trip counter } @@ -478,7 +478,7 @@ PhiNode* PhaseIdealLoop::loop_iv_phi(const Node* xphi, const Node* phi_incr, con PhiNode *phi = xphi->as_Phi(); // Phi must be of loop header; backedge must wrap to increment - if (phi->region() != x) { + if (phi->region() != head) { return nullptr; } return phi; @@ -609,10 +609,10 @@ void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Nod // Find a safepoint node that dominates the back edge. We need a // SafePointNode so we can use its jvm state to create empty // predicates. -static bool no_side_effect_since_safepoint(Compile* C, const Node* x, const Node* mem, MergeMemNode* mm, const PhaseIdealLoop* phase) { +static bool no_side_effect_since_safepoint(Compile* C, const Node* head, const Node* mem, MergeMemNode* mm, const PhaseIdealLoop* phase) { SafePointNode* safepoint = nullptr; - for (DUIterator_Fast imax, i = x->fast_outs(imax); i < imax; i++) { - Node* u = x->fast_out(i); + for (DUIterator_Fast imax, i = head->fast_outs(imax); i < imax; i++) { + Node* u = head->fast_out(i); if (u->is_memory_phi()) { Node* m = u->in(LoopNode::LoopBackControl); if (u->adr_type() == TypePtr::BOTTOM) { @@ -662,14 +662,14 @@ static bool no_side_effect_since_safepoint(Compile* C, const Node* x, const Node return true; } -SafePointNode* PhaseIdealLoop::find_safepoint(Node* back_control, const Node* x, const IdealLoopTree* loop) { +SafePointNode* PhaseIdealLoop::find_safepoint(Node* back_control, const Node* head, const IdealLoopTree* loop) { IfNode* exit_test = back_control->in(0)->as_If(); SafePointNode* safepoint = nullptr; if (exit_test->in(0)->is_SafePoint() && exit_test->in(0)->outcnt() == 1) { safepoint = exit_test->in(0)->as_SafePoint(); } else { Node* c = back_control; - while (c != x && c->Opcode() != Op_SafePoint) { + while (c != head && c->Opcode() != Op_SafePoint) { c = idom(c); } @@ -708,7 +708,7 @@ SafePointNode* PhaseIdealLoop::find_safepoint(Node* back_control, const Node* x, } } #endif - if (!no_side_effect_since_safepoint(C, x, mem, mm, this)) { + if (!no_side_effect_since_safepoint(C, head, mem, mm, this)) { safepoint = nullptr; } else { assert(mm == nullptr|| _igvn.transform(mm) == mem->as_MergeMem()->base_memory(), "all memory state should have been processed"); @@ -1494,7 +1494,7 @@ bool PhaseIdealLoop::CountedLoopConverter::should_stress_long_counted_loop() con return StressLongCountedLoop > 0 && _iv_bt == T_INT && !_head->as_Loop()->is_loop_nest_inner_loop() && - _trunc1 == nullptr; + _increment_truncation_type == TypeInt::INT; // Only stress an int loop (i.e., not char, byte or short) } // Convert an int counted loop to a long counted to stress handling of long counted loops. Returns true upon success. @@ -1648,8 +1648,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { return false; } - CountedLoopNode::TruncatedIncrement - increment = CountedLoopNode::match_incr_with_optional_truncation(iv_incr.incr, _iv_bt); + CountedLoopNode::TruncatedIncrement increment = CountedLoopNode::match_incr_with_optional_truncation(iv_incr.incr, _iv_bt); if (increment.incr == nullptr) { return false; // Funny increment opcode } @@ -2123,7 +2122,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { _stride_con = stride_con; _final_correction = final_correction; _phi = phi; - _phi_incr = iv_incr.phi_incr; + _phi_increment = iv_incr.phi_incr; _stride = stride.stride; _includes_limit = includes_limit; _mask = mask; @@ -2131,7 +2130,7 @@ bool PhaseIdealLoop::CountedLoopConverter::is_counted_loop() { _cmp = exit_test.cmp; _cl_prob = exit_test.cl_prob; _sfpt = sfpt; - _trunc1 = increment.trunc1; + _increment_truncation_type = increment.trunc_type; #ifdef ASSERT _checked_for_counted_loop = true; @@ -2177,7 +2176,7 @@ IdealLoopTree* PhaseIdealLoop::CountedLoopConverter::convert() { } Node* adjusted_limit = _limit; - if (_phi_incr != nullptr) { + if (_phi_increment != nullptr) { // If compare points directly to the phi we need to adjust // the compare so that it points to the incr. Limit have // to be adjusted to keep trip count the same and we diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index ab1dd49adbe71..469637c8dce7d 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1270,7 +1270,7 @@ class PhaseIdealLoop : public PhaseTransform { // Per-Node transform virtual Node* transform(Node* n) { return nullptr; } - Node* loop_exit_control(const Node* x, const IdealLoopTree* loop) const; + Node* loop_exit_control(const Node* head, const IdealLoopTree* loop) const; struct LoopExitTest { Node* cmp = nullptr; @@ -1285,7 +1285,7 @@ class PhaseIdealLoop : public PhaseTransform { Node* incr = nullptr; Node* phi_incr = nullptr; }; - LoopIVIncr loop_iv_incr(Node* old_incr, const Node* x, const IdealLoopTree* loop); + static LoopIVIncr loop_iv_incr(Node* old_incr, const Node* head, const IdealLoopTree* loop); struct LoopIvStride { Node* stride = nullptr; @@ -1293,7 +1293,7 @@ class PhaseIdealLoop : public PhaseTransform { }; static LoopIvStride loop_iv_stride(const Node* incr); - static PhiNode* loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* x); + static PhiNode* loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* head); bool try_convert_to_counted_loop(Node* head, IdealLoopTree*&loop, const BasicType iv_bt); @@ -1301,10 +1301,9 @@ class PhaseIdealLoop : public PhaseTransform { bool create_loop_nest(IdealLoopTree* loop, Node_List &old_new); void add_parse_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop, SafePointNode* sfpt); - SafePointNode* find_safepoint(Node* back_control, const Node* x, const IdealLoopTree* loop); + SafePointNode* find_safepoint(Node* back_control, const Node* head, const IdealLoopTree* loop); IdealLoopTree* insert_outer_loop(IdealLoopTree* loop, LoopNode* outer_l, Node* outer_ift); - IdealLoopTree* create_outer_strip_mined_loop(Node *init_control, - IdealLoopTree* loop, float cl_prob, float le_fcnt, + IdealLoopTree* create_outer_strip_mined_loop(Node* init_control, IdealLoopTree* loop, float cl_prob, float le_fcnt, Node*& entry_control, Node*& iffalse); class CountedLoopConverter { @@ -1324,7 +1323,7 @@ class PhaseIdealLoop : public PhaseTransform { Node* _limit; jlong _stride_con; Node* _phi; - Node* _phi_incr; + Node* _phi_increment; Node* _stride; bool _includes_limit; BoolTest::mask _mask; @@ -1333,7 +1332,7 @@ class PhaseIdealLoop : public PhaseTransform { float _cl_prob; Node* _sfpt; jlong _final_correction; - Node* _trunc1; + const TypeInteger* _increment_truncation_type; public: CountedLoopConverter(PhaseIdealLoop* phase, Node* head, IdealLoopTree* loop, const BasicType iv_bt)