diff --git a/src/hotspot/share/opto/cfgnode.hpp b/src/hotspot/share/opto/cfgnode.hpp index 18d7577d9e6..5edd31fa716 100644 --- a/src/hotspot/share/opto/cfgnode.hpp +++ b/src/hotspot/share/opto/cfgnode.hpp @@ -449,6 +449,10 @@ public: static const TypeInt* filtered_int_type(PhaseGVN* phase, Node* val, Node* if_proj); #ifndef PRODUCT + AssertionPredicateType assertion_predicate_type() const { + return _assertion_predicate_type; + } + virtual void dump_spec(outputStream *st) const; #endif diff --git a/src/hotspot/share/opto/ifnode.cpp b/src/hotspot/share/opto/ifnode.cpp index cedbc66bbb4..8e5cd270213 100644 --- a/src/hotspot/share/opto/ifnode.cpp +++ b/src/hotspot/share/opto/ifnode.cpp @@ -1843,10 +1843,10 @@ void IfProjNode::pin_array_access_nodes(PhaseIterGVN* igvn) { #ifndef PRODUCT void IfNode::dump_spec(outputStream* st) const { switch (_assertion_predicate_type) { - case AssertionPredicateType::Init_value: + case AssertionPredicateType::InitValue: st->print("#Init Value Assertion Predicate "); break; - case AssertionPredicateType::Last_value: + case AssertionPredicateType::LastValue: st->print("#Last Value Assertion Predicate "); break; case AssertionPredicateType::None: diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index b22931e5663..5e585a406f2 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -374,10 +374,10 @@ IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfNod IfProjNode* predicate, Deoptimization::DeoptReason reason, ParsePredicateSuccessProj* parse_predicate_proj) { - TemplateAssertionPredicateExpression template_assertion_predicate_expression( - template_assertion_predicate->in(1)->as_Opaque4()); - Opaque4Node* cloned_opaque4_node = template_assertion_predicate_expression.clone(parse_predicate_proj->in(0)->in(0), this); - IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, template_assertion_predicate->Opcode(), false); + TemplateAssertionExpression template_assertion_expression(template_assertion_predicate->in(1)->as_Opaque4()); + Opaque4Node* cloned_opaque4_node = template_assertion_expression.clone(parse_predicate_proj->in(0)->in(0), this); + IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, + template_assertion_predicate->Opcode(), false); _igvn.replace_input_of(if_proj->in(0), 1, cloned_opaque4_node); _igvn.replace_input_of(parse_predicate_proj->in(0), 0, if_proj); set_idom(parse_predicate_proj->in(0), if_proj, dom_depth(if_proj)); @@ -1324,7 +1324,7 @@ IfTrueNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL C->add_template_assertion_predicate_opaq(opaque_bol); register_new_node(opaque_bol, upper_bound_proj); IfTrueNode* new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode(), - false NOT_PRODUCT(COMMA AssertionPredicateType::Init_value)); + false NOT_PRODUCT(COMMA AssertionPredicateType::InitValue)); _igvn.replace_input_of(new_proj->in(0), 1, opaque_bol); assert(opaque_init->outcnt() > 0, "should be used"); @@ -1350,7 +1350,7 @@ IfTrueNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL C->add_template_assertion_predicate_opaq(opaque_bol); register_new_node(opaque_bol, new_proj); new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode(), - false NOT_PRODUCT(COMMA AssertionPredicateType::Last_value)); + false NOT_PRODUCT(COMMA AssertionPredicateType::LastValue)); _igvn.replace_input_of(new_proj->in(0), 1, opaque_bol); assert(max_value->outcnt() > 0, "should be used"); assert(assertion_predicate_has_loop_opaque_node(new_proj->in(0)->as_If()), "unexpected"); diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 6ca7bb51fb4..99742a598e8 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -1373,17 +1373,12 @@ void PhaseIdealLoop::copy_assertion_predicates_to_main_loop_helper(const Predica Node* bol = iff->in(1); assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate"); if (bol->is_Opaque4()) { - assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes"); // Clone the Assertion Predicate twice and initialize one with the initial // value of the loop induction variable. Leave the other predicate // to be initialized when increasing the stride during loop unrolling. - prev_proj = clone_assertion_predicate_and_initialize(iff, opaque_init, nullptr, predicate_proj, uncommon_proj, - current_proj, outer_loop, prev_proj); - assert(assertion_predicate_has_loop_opaque_node(prev_proj->in(0)->as_If()), ""); - - prev_proj = clone_assertion_predicate_and_initialize(iff, init, stride, predicate_proj, uncommon_proj, - current_proj, outer_loop, prev_proj); - assert(!assertion_predicate_has_loop_opaque_node(prev_proj->in(0)->as_If()), ""); + prev_proj = clone_template_assertion_predicate(iff, opaque_init, predicate_proj, uncommon_proj, + current_proj, outer_loop, prev_proj); + prev_proj = create_initialized_assertion_predicate(iff, init, stride, prev_proj); // Rewire any control inputs from the cloned Assertion Predicates down to the main and post loop for data nodes // that are part of the main loop (and were cloned to the pre and post loop). @@ -1460,7 +1455,7 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) wq.push(n); for (uint i = 0; i < wq.size(); i++) { Node* n = wq.at(i); - if (TemplateAssertionPredicateExpressionNode::is_maybe_in_expression(n)) { + if (TemplateAssertionExpressionNode::is_maybe_in_expression(n)) { if (n->is_OpaqueLoopInit()) { init++; } else if (n->is_OpaqueLoopStride()) { @@ -1477,27 +1472,28 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) } } -// Clone an Assertion Predicate for the main loop. new_init and new_stride are set as new inputs. Since the predicates -// cannot fail at runtime, Halt nodes are inserted instead of uncommon traps. -Node* PhaseIdealLoop::clone_assertion_predicate_and_initialize(Node* iff, Node* new_init, Node* new_stride, Node* predicate, - Node* uncommon_proj, Node* control, IdealLoopTree* outer_loop, - Node* input_proj) { - TemplateAssertionPredicateExpression template_assertion_predicate_expression(iff->in(1)->as_Opaque4()); - Node* new_opaque_node; - if (new_stride == nullptr) { - // Clone the Template Assertion Predicate and set a new OpaqueLoopInitNode to create a new Template Assertion Predicate. - // This is done when creating a new Template Assertion Predicate for the main loop which requires a new init node. - // We keep the Opaque4 node since it's still a template. - assert(new_init->is_OpaqueLoopInit(), "only for creating new Template Assertion Predicates"); - new_opaque_node = template_assertion_predicate_expression.clone_and_replace_init(new_init, control, this); - } else { - // Create an Initialized Assertion Predicate from the Template Assertion Predicate. - new_opaque_node = template_assertion_predicate_expression.clone_and_replace_init_and_stride(new_init, new_stride, - control, this); - // Since this is an Initialized Assertion Predicate, we use the dedicated opaque node. - new_opaque_node = new OpaqueInitializedAssertionPredicateNode(new_opaque_node->in(1)->as_Bool(), C); - register_new_node(new_opaque_node, control); - } +// Create an Initialized Assertion Predicate from the template_assertion_predicate +IfTrueNode* PhaseIdealLoop::create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init, + Node* new_stride, Node* control) { + assert(assertion_predicate_has_loop_opaque_node(template_assertion_predicate), + "must find OpaqueLoop* nodes for Template Assertion Predicate"); + InitializedAssertionPredicate initialized_assertion_predicate(template_assertion_predicate, new_init, new_stride, this); + IfTrueNode* success_proj = initialized_assertion_predicate.create(control); + assert(!assertion_predicate_has_loop_opaque_node(success_proj->in(0)->as_If()), + "Initialized Assertion Predicates do not have OpaqueLoop* nodes in the bool expression anymore"); + return success_proj; +} + +// Clone the Template Assertion Predicate and set a new OpaqueLoopInitNode to create a new Template Assertion Predicate. +// This is done when creating a new Template Assertion Predicate for the main loop which requires a new init node. +// We keep the Opaque4 node since it's still a template. Since the templates are eventually removed after loop opts, +// these are never executed. We therefore insert a Halt node instead of an uncommon trap. +Node* PhaseIdealLoop::clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, + Node* control, IdealLoopTree* outer_loop, Node* input_proj) { + assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes for Template Assertion Predicate"); + TemplateAssertionExpression template_assertion_expression(iff->in(1)->as_Opaque4()); + assert(new_init->is_OpaqueLoopInit(), "only for creating new Template Assertion Predicates"); + Opaque4Node* new_opaque_node = template_assertion_expression.clone_and_replace_init(new_init, control, this); Node* proj = predicate->clone(); Node* other_proj = uncommon_proj->clone(); Node* new_iff = iff->clone(); @@ -1506,8 +1502,7 @@ Node* PhaseIdealLoop::clone_assertion_predicate_and_initialize(Node* iff, Node* other_proj->set_req(0, new_iff); Node* frame = new ParmNode(C->start(), TypeFunc::FramePtr); register_new_node(frame, C->start()); - // It's impossible for the predicate to fail at runtime. Use a Halt node. - Node* halt = new HaltNode(other_proj, frame, "duplicated predicate failed which is impossible"); + Node* halt = new HaltNode(other_proj, frame, "Template Assertion Predicates are always removed before code generation"); _igvn.add_input_to(C->root(), halt); new_iff->set_req(0, input_proj); @@ -1515,6 +1510,8 @@ Node* PhaseIdealLoop::clone_assertion_predicate_and_initialize(Node* iff, Node* register_control(proj, outer_loop == _ltree_root ? _ltree_root : outer_loop->_parent, new_iff); register_control(other_proj, _ltree_root, new_iff); register_control(halt, _ltree_root, other_proj); + assert(assertion_predicate_has_loop_opaque_node(proj->in(0)->as_If()), + "Template Assertion Predicates must have OpaqueLoop* nodes in the bool expression"); return proj; } @@ -1967,9 +1964,7 @@ void PhaseIdealLoop::update_main_loop_assertion_predicates(Node* ctrl, CountedLo // Create an Initialized Assertion Predicates for it accordingly: // - For the initial access a[init] (same as before) // - For the last access a[init+new_stride-orig_stride] (with the new unroll stride) - prev_proj = clone_assertion_predicate_and_initialize(iff, init, max_value, entry, proj, ctrl, outer_loop, - prev_proj); - assert(!assertion_predicate_has_loop_opaque_node(prev_proj->in(0)->as_If()), "unexpected"); + prev_proj = create_initialized_assertion_predicate(iff, init, max_value, prev_proj); } else { // Ignore Opaque4 from a non-null-check for an intrinsic or unsafe access. This could happen when we maximally // unroll a non-main loop with such an If with an Opaque4 node directly above the loop entry. @@ -2008,10 +2003,7 @@ void PhaseIdealLoop::copy_assertion_predicates_to_post_loop(LoopNode* main_loop_ } if (iff->in(1)->is_Opaque4()) { // Initialize from Template Assertion Predicate. - assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes"); - prev_proj = clone_assertion_predicate_and_initialize(iff, init, stride, ctrl, proj, post_loop_entry, - post_loop, prev_proj); - assert(!assertion_predicate_has_loop_opaque_node(prev_proj->in(0)->as_If()), "must not find OpaqueLoop* nodes"); + prev_proj = create_initialized_assertion_predicate(iff, init, stride, prev_proj); } ctrl = ctrl->in(0)->in(0); } @@ -2030,9 +2022,7 @@ void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(const Predi if (!predicate_block->has_parse_predicate()) { return; } - Node* control = outer_loop_head->in(LoopNode::EntryControl); - Node* input_proj = control; - + Node* input_proj = outer_loop_head->in(LoopNode::EntryControl); const Node* parse_predicate_uncommon_trap = predicate_block->parse_predicate()->uncommon_trap(); Node* next_regular_predicate_proj = predicate_block->skip_parse_predicate(); while (next_regular_predicate_proj->is_IfProj()) { @@ -2046,9 +2036,7 @@ void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(const Predi assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate"); if (bol->is_Opaque4()) { // Initialize from Template Assertion Predicate. - assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes"); - input_proj = clone_assertion_predicate_and_initialize(iff, init, stride, next_regular_predicate_proj, uncommon_proj, control, - outer_loop, input_proj); + input_proj = create_initialized_assertion_predicate(iff, init, stride, input_proj); // Rewire any control inputs from the old Assertion Predicates above the peeled iteration down to the initialized // Assertion Predicates above the peeled loop. @@ -3018,7 +3006,7 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { // unrolling or splitting this main-loop further. loop_entry = add_range_check_elimination_assertion_predicate( loop, loop_entry, scale_con, int_offset, int_limit, stride_con, opaque_init, true - NOT_PRODUCT(COMMA AssertionPredicateType::Init_value)); + NOT_PRODUCT(COMMA AssertionPredicateType::InitValue)); assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); Node* opaque_stride = new OpaqueLoopStrideNode(C, cl->stride()); @@ -3032,7 +3020,7 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { register_new_node(max_value, loop_entry); loop_entry = add_range_check_elimination_assertion_predicate( loop, loop_entry, scale_con, int_offset, int_limit, stride_con, max_value, true - NOT_PRODUCT(COMMA AssertionPredicateType::Last_value)); + NOT_PRODUCT(COMMA AssertionPredicateType::LastValue)); assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); } else { diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 24101ea07a0..94632be268d 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -952,9 +952,10 @@ private: LoopNode* outer_main_head, uint dd_main_head, uint idx_before_pre_post, uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List& old_new); - Node* clone_assertion_predicate_and_initialize(Node* iff, Node* new_init, Node* new_stride, Node* predicate, - Node* uncommon_proj, Node* control, IdealLoopTree* outer_loop, - Node* input_proj); + Node* clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, Node* control, + IdealLoopTree* outer_loop, Node* input_proj); + IfTrueNode* create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init, + Node* new_stride, Node* control); static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride); static bool assertion_predicate_has_loop_opaque_node(IfNode* iff); static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false); @@ -1774,7 +1775,7 @@ public: bool clone_cmp_loadklass_down(Node* n, const Node* blk1, const Node* blk2); void clone_loadklass_nodes_at_cmp_index(const Node* n, Node* cmp, int i); bool clone_cmp_down(Node* n, const Node* blk1, const Node* blk2); - void clone_template_assertion_predicate_expression_down(Node* node); + void clone_template_assertion_expression_down(Node* node); Node* similar_subtype_check(const Node* x, Node* r_in); diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index 5b0de2e02d5..3887e8a5f6c 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -27,6 +27,7 @@ #include "opto/loopnode.hpp" #include "opto/node.hpp" #include "opto/predicates.hpp" +#include "opto/rootnode.hpp" // Walk over all Initialized Assertion Predicates and return the entry into the first Initialized Assertion Predicate // (i.e. not belonging to an Initialized Assertion Predicate anymore) @@ -239,27 +240,27 @@ class ReplaceInitAndStrideStrategy : public TransformStrategyForOpaqueLoopNodes } }; -// Creates an identical clone of this Template Assertion Predicate Expression (i.e.cloning all nodes from the Opaque4Node -// to and including the OpaqueLoop* nodes). The cloned nodes are rewired to reflect the same graph structure as found for -// this Template Assertion Predicate Expression. The cloned nodes get 'new_ctrl' as ctrl. There is no other update done -// for the cloned nodes. Return the newly cloned Opaque4Node. -Opaque4Node* TemplateAssertionPredicateExpression::clone(Node* new_ctrl, PhaseIdealLoop* phase) { +// Creates an identical clone of this Template Assertion Expression (i.e.cloning all nodes from the Opaque4Node to and +// including the OpaqueLoop* nodes). The cloned nodes are rewired to reflect the same graph structure as found for this +// Template Assertion Expression. The cloned nodes get 'new_ctrl' as ctrl. There is no other update done for the cloned +// nodes. Return the newly cloned Opaque4Node. +Opaque4Node* TemplateAssertionExpression::clone(Node* new_ctrl, PhaseIdealLoop* phase) { CloneStrategy clone_init_and_stride_strategy(phase, new_ctrl); return clone(clone_init_and_stride_strategy, new_ctrl, phase); } // Same as clone() but instead of cloning the OpaqueLoopInitNode, we replace it with the provided 'new_init' node. -Opaque4Node* TemplateAssertionPredicateExpression::clone_and_replace_init(Node* new_init, Node* new_ctrl, - PhaseIdealLoop* phase) { +Opaque4Node* TemplateAssertionExpression::clone_and_replace_init(Node* new_init, Node* new_ctrl, + PhaseIdealLoop* phase) { ReplaceInitAndCloneStrideStrategy replace_init_and_clone_stride_strategy(new_init, new_ctrl, phase); return clone(replace_init_and_clone_stride_strategy, new_ctrl, phase); } // Same as clone() but instead of cloning the OpaqueLoopInit and OpaqueLoopStride node, we replace them with the provided // 'new_init' and 'new_stride' nodes, respectively. -Opaque4Node* TemplateAssertionPredicateExpression::clone_and_replace_init_and_stride(Node* new_init, Node* new_stride, - Node* new_ctrl, - PhaseIdealLoop* phase) { +Opaque4Node* TemplateAssertionExpression::clone_and_replace_init_and_stride(Node* new_init, Node* new_stride, + Node* new_ctrl, + PhaseIdealLoop* phase) { ReplaceInitAndStrideStrategy replace_init_and_stride_strategy(new_init, new_stride); return clone(replace_init_and_stride_strategy, new_ctrl, phase); } @@ -307,8 +308,7 @@ class DataNodesOnPathsToTargets : public StackObj { // Do a BFS from the start_node to collect all target nodes. We can then do another BFS from the target nodes to // find all nodes on the paths from start->target(s). // Note: We could do a single DFS pass to search targets and backtrack in one walk. But this is much more complex. - // Given that the typical Template Assertion Predicate Expression only consists of a few nodes, we aim for - // simplicity here. + // Given that the typical Template Assertion Expression only consists of a few nodes, we aim for simplicity here. void collect_target_nodes(Node* start_node) { _nodes_to_visit.push(start_node); for (uint i = 0; i < _nodes_to_visit.size(); i++) { @@ -342,14 +342,14 @@ class DataNodesOnPathsToTargets : public StackObj { } }; -// Clones this Template Assertion Predicate Expression and applies the given strategy to transform the OpaqueLoop* nodes. -Opaque4Node* TemplateAssertionPredicateExpression::clone(const TransformStrategyForOpaqueLoopNodes& transform_strategy, - Node* new_ctrl, PhaseIdealLoop* phase) { +// Clones this Template Assertion Expression and applies the given strategy to transform the OpaqueLoop* nodes. +Opaque4Node* TemplateAssertionExpression::clone(const TransformStrategyForOpaqueLoopNodes& transform_strategy, + Node* new_ctrl, PhaseIdealLoop* phase) { ResourceMark rm; auto is_opaque_loop_node = [](const Node* node) { return node->is_Opaque1(); }; - DataNodesOnPathsToTargets data_nodes_on_path_to_targets(TemplateAssertionPredicateExpressionNode::is_maybe_in_expression, + DataNodesOnPathsToTargets data_nodes_on_path_to_targets(TemplateAssertionExpressionNode::is_maybe_in_expression, is_opaque_loop_node); const Unique_Node_List& collected_nodes = data_nodes_on_path_to_targets.collect(_opaque4_node); DataNodeGraph data_node_graph(collected_nodes, phase); @@ -359,8 +359,8 @@ Opaque4Node* TemplateAssertionPredicateExpression::clone(const TransformStrategy return opaque4_clone->as_Opaque4(); } -// Check if this node belongs a Template Assertion Predicate Expression (including OpaqueLoop* nodes). -bool TemplateAssertionPredicateExpressionNode::is_in_expression(Node* node) { +// Check if this node belongs a Template Assertion Expression (including OpaqueLoop* nodes). +bool TemplateAssertionExpressionNode::is_in_expression(Node* node) { if (is_maybe_in_expression(node)) { ResourceMark rm; Unique_Node_List list; @@ -377,10 +377,90 @@ bool TemplateAssertionPredicateExpressionNode::is_in_expression(Node* node) { return false; } -bool TemplateAssertionPredicateExpressionNode::is_template_assertion_predicate(Node* node) { +bool TemplateAssertionExpressionNode::is_template_assertion_predicate(Node* node) { return node->is_If() && node->in(1)->is_Opaque4(); } +InitializedAssertionPredicate::InitializedAssertionPredicate(IfNode* template_assertion_predicate, Node* new_init, + Node* new_stride, PhaseIdealLoop* phase) + : _template_assertion_predicate(template_assertion_predicate), + _new_init(new_init), + _new_stride(new_stride), + _phase(phase) {} + +// Create an Initialized Assertion Predicate at the provided control from the _template_assertion_predicate. +// We clone the Template Assertion Expression and replace: +// - Opaque4 with OpaqueInitializedAssertionPredicate +// - OpaqueLoop*Nodes with _new_init and _new_stride, respectively. +// +// / init stride +// | | | +// | OpaqueLoopInitNode OpaqueLoopStrideNode / _new_init _new_stride +// Template | \ / | \ / +// Assertion | ... Assertion | ... +// Expression | | Expression | | +// | Bool | new Bool +// | | | | +// \ Opaque4 ======> control \ OpaqueInitializedAssertionPredicate +// | \ / +// If new If +// / \ / \ +// success fail path new success new Halt +// proj (Halt or UCT) proj +// +IfTrueNode* InitializedAssertionPredicate::create(Node* control) { + IdealLoopTree* loop = _phase->get_loop(control); + OpaqueInitializedAssertionPredicateNode* assertion_expression = create_assertion_expression(control); + IfNode* if_node = create_if_node(control, assertion_expression, loop); + create_fail_path(if_node, loop); + return create_success_path(if_node, loop); +} + +// Create a new Assertion Expression to be used as bool input for the Initialized Assertion Predicate IfNode. +OpaqueInitializedAssertionPredicateNode* InitializedAssertionPredicate::create_assertion_expression(Node* control) { + Opaque4Node* template_opaque = _template_assertion_predicate->in(1)->as_Opaque4(); + TemplateAssertionExpression template_assertion_expression(template_opaque); + Opaque4Node* tmp_opaque = template_assertion_expression.clone_and_replace_init_and_stride(_new_init, _new_stride, + control, _phase); + OpaqueInitializedAssertionPredicateNode* assertion_expression = + new OpaqueInitializedAssertionPredicateNode(tmp_opaque->in(1)->as_Bool(), _phase->C); + _phase->register_new_node(assertion_expression, control); + return assertion_expression; +} + +IfNode* InitializedAssertionPredicate::create_if_node(Node* control, + OpaqueInitializedAssertionPredicateNode* assertion_expression, + IdealLoopTree* loop) { + const int if_opcode = _template_assertion_predicate->Opcode(); + NOT_PRODUCT(const AssertionPredicateType assertion_predicate_type = _template_assertion_predicate->assertion_predicate_type();) + IfNode* if_node = if_opcode == Op_If ? + new IfNode(control, assertion_expression, PROB_MAX, COUNT_UNKNOWN NOT_PRODUCT(COMMA assertion_predicate_type)) : + new RangeCheckNode(control, assertion_expression, PROB_MAX, COUNT_UNKNOWN NOT_PRODUCT(COMMA assertion_predicate_type)); + _phase->register_control(if_node, loop, control); + return if_node; +} + +IfTrueNode* InitializedAssertionPredicate::create_success_path(IfNode* if_node, IdealLoopTree* loop) { + IfTrueNode* success_proj = new IfTrueNode(if_node); + _phase->register_control(success_proj, loop, if_node); + return success_proj; +} + +void InitializedAssertionPredicate::create_fail_path(IfNode* if_node, IdealLoopTree* loop) { + IfFalseNode* fail_proj = new IfFalseNode(if_node); + _phase->register_control(fail_proj, loop, if_node); + create_halt_node(fail_proj, loop); +} + +void InitializedAssertionPredicate::create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop) { + StartNode* start_node = _phase->C->start(); + Node* frame = new ParmNode(start_node, TypeFunc::FramePtr); + _phase->register_new_node(frame, start_node); + Node* halt = new HaltNode(fail_proj, frame, "Initialized Assertion Predicate cannot fail"); + _phase->igvn().add_input_to(_phase->C->root(), halt); + _phase->register_control(halt, loop, fail_proj); +} + // Is current node pointed to by iterator a predicate? bool PredicateEntryIterator::has_next() const { return ParsePredicate::is_predicate(_current) || diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index 08aa64f03e5..96f5c438b80 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -29,6 +29,8 @@ #include "opto/connode.hpp" #include "opto/opaquenode.hpp" +class IdealLoopTree; + /* * There are different kinds of predicates throughout the code. We differentiate between the following predicates: * @@ -198,8 +200,8 @@ // value of a range check in the last iteration of a loop. enum class AssertionPredicateType { None, // Not an Assertion Predicate - Init_value, - Last_value + InitValue, + LastValue }; #endif // NOT PRODUCT @@ -294,20 +296,20 @@ class RuntimePredicate : public StackObj { static bool is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason); }; -// Interface to transform OpaqueLoopInit and OpaqueLoopStride nodes of a Template Assertion Predicate Expression. +// Interface to transform OpaqueLoopInit and OpaqueLoopStride nodes of a Template Assertion Expression. class TransformStrategyForOpaqueLoopNodes : public StackObj { public: virtual Node* transform_opaque_init(OpaqueLoopInitNode* opaque_init) const = 0; virtual Node* transform_opaque_stride(OpaqueLoopStrideNode* opaque_stride) const = 0; }; -// A Template Assertion Predicate Expression represents the Opaque4Node for the initial value or the last value of a +// A Template Assertion Predicate represents the Opaque4Node for the initial value or the last value of a // Template Assertion Predicate and all the nodes up to and including the OpaqueLoop* nodes. -class TemplateAssertionPredicateExpression : public StackObj { +class TemplateAssertionExpression : public StackObj { Opaque4Node* _opaque4_node; public: - explicit TemplateAssertionPredicateExpression(Opaque4Node* opaque4_node) : _opaque4_node(opaque4_node) {} + explicit TemplateAssertionExpression(Opaque4Node* opaque4_node) : _opaque4_node(opaque4_node) {} private: Opaque4Node* clone(const TransformStrategyForOpaqueLoopNodes& transform_strategy, Node* new_ctrl, PhaseIdealLoop* phase); @@ -318,29 +320,29 @@ class TemplateAssertionPredicateExpression : public StackObj { Opaque4Node* clone_and_replace_init_and_stride(Node* new_init, Node* new_stride, Node* new_ctrl, PhaseIdealLoop* phase); }; -// Class to represent a node being part of a Template Assertion Predicate Expression. +// Class to represent a node being part of a Template Assertion Expression. Note that this is not an IR node. // // The expression itself can belong to no, one, or two Template Assertion Predicates: // - None: This node is already dead (i.e. we replaced the Bool condition of the Template Assertion Predicate). // - Two: A OpaqueLoopInitNode could be part of two Template Assertion Predicates. // - One: In all other cases. -class TemplateAssertionPredicateExpressionNode : public StackObj { +class TemplateAssertionExpressionNode : public StackObj { Node* const _node; public: - explicit TemplateAssertionPredicateExpressionNode(Node* node) : _node(node) { + explicit TemplateAssertionExpressionNode(Node* node) : _node(node) { assert(is_in_expression(node), "must be valid"); } - NONCOPYABLE(TemplateAssertionPredicateExpressionNode); + NONCOPYABLE(TemplateAssertionExpressionNode); private: static bool is_template_assertion_predicate(Node* node); public: - // Check whether the provided node is part of a Template Assertion Predicate Expression or not. + // Check whether the provided node is part of a Template Assertion Expression or not. static bool is_in_expression(Node* node); - // Check if the opcode of node could be found in a Template Assertion Predicate Expression. + // Check if the opcode of node could be found in a Template Assertion Expression. // This also provides a fast check whether a node is unrelated. static bool is_maybe_in_expression(const Node* node) { const int opcode = node->Opcode(); @@ -377,21 +379,44 @@ class TemplateAssertionPredicateExpressionNode : public StackObj { callback(next->as_If()); DEBUG_ONLY(template_counter++;) } else { - assert(!next->is_CFG(), "no CFG expected in Template Assertion Predicate Expression"); + assert(!next->is_CFG(), "no CFG expected in Template Assertion Expression"); list.push_outputs_of(next); } } - // Each node inside a Template Assertion Predicate Expression is in between a Template Assertion Predicate and - // its OpaqueLoop* nodes (or an OpaqueLoop* node itself). The OpaqueLoop* nodes do not common up. Therefore, each - // Template Assertion Predicate Expression node belongs to a single expression - except for OpaqueLoopInitNodes. - // An OpaqueLoopInitNode is shared between the init and last value Template Assertion Predicate at creation. - // Later, when cloning the expressions, they are no longer shared. + // Each node inside a Template Assertion Expression is in between a Template Assertion Predicate and its OpaqueLoop* + // nodes (or an OpaqueLoop* node itself). The OpaqueLoop* nodes do not common up. Therefore, each Template Assertion + // Expression node belongs to a single expression - except for OpaqueLoopInitNodes. An OpaqueLoopInitNode is shared + // between the init and last value Template Assertion Predicate at creation. Later, when cloning the expressions, + // they are no longer shared. assert(template_counter <= 2, "a node cannot be part of more than two templates"); assert(template_counter <= 1 || _node->is_OpaqueLoopInit(), "only OpaqueLoopInit nodes can be part of two templates"); } }; +// This class creates a new Initialized Assertion Predicate. +class InitializedAssertionPredicate : public StackObj { + IfNode* const _template_assertion_predicate; + Node* const _new_init; + Node* const _new_stride; + PhaseIdealLoop* const _phase; + + public: + InitializedAssertionPredicate(IfNode* template_assertion_predicate, Node* new_init, Node* new_stride, + PhaseIdealLoop* phase); + NONCOPYABLE(InitializedAssertionPredicate); + + IfTrueNode* create(Node* control); + + private: + OpaqueInitializedAssertionPredicateNode* create_assertion_expression(Node* control); + IfNode* create_if_node(Node* control, OpaqueInitializedAssertionPredicateNode* assertion_expression, IdealLoopTree* loop); + void create_fail_path(IfNode* if_node, IdealLoopTree* loop); + void create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop); + IfTrueNode* create_success_path(IfNode* if_node, IdealLoopTree* loop); +}; + + // This class represents a Predicate Block (i.e. either a Loop Predicate Block, a Profiled Loop Predicate Block, // or a Loop Limit Check Predicate Block). It contains zero or more Regular Predicates followed by a Parse Predicate // which, however, does not need to exist (we could already have decided to remove Parse Predicates for this loop). diff --git a/src/hotspot/share/opto/split_if.cpp b/src/hotspot/share/opto/split_if.cpp index ad4053e2c98..1eff44ab783 100644 --- a/src/hotspot/share/opto/split_if.cpp +++ b/src/hotspot/share/opto/split_if.cpp @@ -95,7 +95,7 @@ bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) { return true; } - clone_template_assertion_predicate_expression_down(n); + clone_template_assertion_expression_down(n); if (n->Opcode() == Op_OpaqueZeroTripGuard) { // If this Opaque1 is part of the zero trip guard for a loop: @@ -409,25 +409,25 @@ bool PhaseIdealLoop::clone_cmp_down(Node* n, const Node* blk1, const Node* blk2) return false; } -// 'n' could be a node belonging to a Template Assertion Predicate Expression (i.e. any node between a Template -// Assertion Predicate and its OpaqueLoop* nodes (included)). We cannot simply split this node up since this would -// create a phi node inside the Template Assertion Predicate Expression - making it unrecognizable as such. Therefore, -// we completely clone the entire Template Assertion Predicate Expression "down". This ensures that we have an -// untouched copy that is still recognized by the Template Assertion Predicate matching code. -void PhaseIdealLoop::clone_template_assertion_predicate_expression_down(Node* node) { - if (!TemplateAssertionPredicateExpressionNode::is_in_expression(node)) { +// 'n' could be a node belonging to a Template Assertion Expression (i.e. any node between a Template Assertion Predicate +// and its OpaqueLoop* nodes (included)). We cannot simply split this node up since this would create a phi node inside +// the Template Assertion Expression - making it unrecognizable as such. Therefore, we completely clone the entire +// Template Assertion Expression "down". This ensures that we have an untouched copy that is still recognized by the +// Template Assertion Predicate matching code. +void PhaseIdealLoop::clone_template_assertion_expression_down(Node* node) { + if (!TemplateAssertionExpressionNode::is_in_expression(node)) { return; } - TemplateAssertionPredicateExpressionNode template_assertion_predicate_expression_node(node); + TemplateAssertionExpressionNode template_assertion_expression_node(node); auto clone_expression = [&](IfNode* template_assertion_predicate) { Opaque4Node* opaque4_node = template_assertion_predicate->in(1)->as_Opaque4(); - TemplateAssertionPredicateExpression template_assertion_predicate_expression(opaque4_node); + TemplateAssertionExpression template_assertion_expression(opaque4_node); Node* new_ctrl = template_assertion_predicate->in(0); - Opaque4Node* cloned_opaque4_node = template_assertion_predicate_expression.clone(new_ctrl, this); + Opaque4Node* cloned_opaque4_node = template_assertion_expression.clone(new_ctrl, this); igvn().replace_input_of(template_assertion_predicate, 1, cloned_opaque4_node); }; - template_assertion_predicate_expression_node.for_each_template_assertion_predicate(clone_expression); + template_assertion_expression_node.for_each_template_assertion_predicate(clone_expression); } //------------------------------register_new_node------------------------------