Skip to content

Commit 3518b4b

Browse files
chhagedornchhagedorn
committedDec 16, 2024
8344171: Clone and initialize Assertion Predicates in order instead of in reverse-order
Reviewed-by: epeter, kvn
1 parent c88e081 commit 3518b4b

File tree

4 files changed

+78
-46
lines changed

4 files changed

+78
-46
lines changed
 

‎src/hotspot/share/opto/loopTransform.cpp

+2-11
Original file line numberDiff line numberDiff line change
@@ -1748,20 +1748,11 @@ void PhaseIdealLoop::create_assertion_predicates_at_loop(CountedLoopNode* source
17481748
CountedLoopNode* target_loop_head,
17491749
const NodeInLoopBody& _node_in_loop_body,
17501750
const bool clone_template) {
1751-
Node* init = target_loop_head->init_trip();
1752-
Node* stride = target_loop_head->stride();
1753-
LoopNode* target_outer_loop_head = target_loop_head->skip_strip_mined();
1754-
Node* target_loop_entry = target_outer_loop_head->in(LoopNode::EntryControl);
1755-
CreateAssertionPredicatesVisitor create_assertion_predicates_visitor(init, stride, target_loop_entry, this,
1756-
_node_in_loop_body, clone_template);
1751+
CreateAssertionPredicatesVisitor create_assertion_predicates_visitor(target_loop_head, this, _node_in_loop_body,
1752+
clone_template);
17571753
Node* source_loop_entry = source_loop_head->skip_strip_mined()->in(LoopNode::EntryControl);
17581754
PredicateIterator predicate_iterator(source_loop_entry);
17591755
predicate_iterator.for_each(create_assertion_predicates_visitor);
1760-
if (create_assertion_predicates_visitor.has_created_predicates()) {
1761-
IfTrueNode* last_created_predicate_success_proj = create_assertion_predicates_visitor.last_created_success_proj();
1762-
_igvn.replace_input_of(target_outer_loop_head, LoopNode::EntryControl, last_created_predicate_success_proj);
1763-
set_idom(target_outer_loop_head, last_created_predicate_success_proj, dom_depth(target_outer_loop_head));
1764-
}
17651756
}
17661757
//------------------------------do_unroll--------------------------------------
17671758
// Unroll the loop body one step - make each trip do 2 iterations.

‎src/hotspot/share/opto/loopnode.hpp

+6
Original file line numberDiff line numberDiff line change
@@ -1367,6 +1367,12 @@ class PhaseIdealLoop : public PhaseTransform {
13671367
public:
13681368
void register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body = true);
13691369

1370+
// Replace the control input of 'node' with 'new_control' and set the dom depth to the one of 'new_control'.
1371+
void replace_control(Node* node, Node* new_control) {
1372+
_igvn.replace_input_of(node, 0, new_control);
1373+
set_idom(node, new_control, dom_depth(new_control));
1374+
}
1375+
13701376
void replace_loop_entry(LoopNode* loop_head, Node* new_entry) {
13711377
_igvn.replace_input_of(loop_head, LoopNode::EntryControl, new_entry);
13721378
set_idom(loop_head, new_entry, dom_depth(new_entry));

‎src/hotspot/share/opto/predicates.cpp

+64-9
Original file line numberDiff line numberDiff line change
@@ -878,6 +878,19 @@ void Predicates::dump_for_loop(LoopNode* loop_node) {
878878
}
879879
#endif // NOT PRODUCT
880880

881+
CreateAssertionPredicatesVisitor::CreateAssertionPredicatesVisitor(CountedLoopNode* target_loop_head,
882+
PhaseIdealLoop* phase,
883+
const NodeInLoopBody& node_in_loop_body,
884+
const bool clone_template)
885+
: _init(target_loop_head->init_trip()),
886+
_stride(target_loop_head->stride()),
887+
_old_target_loop_entry(target_loop_head->skip_strip_mined()->in(LoopNode::EntryControl)),
888+
_current_predicate_chain_head(target_loop_head->skip_strip_mined()), // Initially no predicates, yet.
889+
_phase(phase),
890+
_has_hoisted_check_parse_predicates(false),
891+
_node_in_loop_body(node_in_loop_body),
892+
_clone_template(clone_template) {}
893+
881894
// Keep track of whether we are in the correct Predicate Block where Template Assertion Predicates can be found.
882895
// The PredicateIterator will always start at the loop entry and first visits the Loop Limit Check Predicate Block.
883896
void CreateAssertionPredicatesVisitor::visit(const ParsePredicate& parse_predicate) {
@@ -894,31 +907,74 @@ void CreateAssertionPredicatesVisitor::visit(const TemplateAssertionPredicate& t
894907
return;
895908
}
896909
if (_clone_template) {
897-
_new_control = clone_template_and_replace_init_input(template_assertion_predicate);
910+
IfTrueNode* cloned_template_success_proj = clone_template_and_replace_init_input(template_assertion_predicate);
911+
initialize_from_template(template_assertion_predicate, cloned_template_success_proj);
912+
_current_predicate_chain_head = cloned_template_success_proj->in(0);
913+
} else {
914+
IfTrueNode* initialized_success_proj = initialize_from_template(template_assertion_predicate, _old_target_loop_entry);
915+
_current_predicate_chain_head = initialized_success_proj->in(0);
898916
}
899-
_new_control = initialize_from_template(template_assertion_predicate);
900917
}
901918

902919
// Create an Initialized Assertion Predicate from the provided Template Assertion Predicate.
903920
IfTrueNode* CreateAssertionPredicatesVisitor::initialize_from_template(
904-
const TemplateAssertionPredicate& template_assertion_predicate) const {
921+
const TemplateAssertionPredicate& template_assertion_predicate, Node* new_control) const {
905922
DEBUG_ONLY(template_assertion_predicate.verify();)
906923
IfNode* template_head = template_assertion_predicate.head();
907924
InitializedAssertionPredicateCreator initialized_assertion_predicate_creator(_phase);
908925
IfTrueNode* initialized_predicate = initialized_assertion_predicate_creator.create_from_template(template_head,
909-
_new_control,
926+
new_control,
910927
_init, _stride);
911928
DEBUG_ONLY(InitializedAssertionPredicate::verify(initialized_predicate);)
912929
template_assertion_predicate.rewire_loop_data_dependencies(initialized_predicate, _node_in_loop_body, _phase);
930+
rewire_to_old_predicate_chain_head(initialized_predicate);
913931
return initialized_predicate;
914932
}
915933

916934
// Clone the provided 'template_assertion_predicate' and set '_init' as new input for the OpaqueLoopInitNode.
917935
IfTrueNode* CreateAssertionPredicatesVisitor::clone_template_and_replace_init_input(
918936
const TemplateAssertionPredicate& template_assertion_predicate) {
919937
OpaqueLoopInitNode* opaque_init = new OpaqueLoopInitNode(_phase->C, _init);
920-
_phase->register_new_node(opaque_init, _new_control);
921-
return template_assertion_predicate.clone_and_replace_init(_new_control, opaque_init, _phase);
938+
_phase->register_new_node(opaque_init, _old_target_loop_entry);
939+
return template_assertion_predicate.clone_and_replace_init(_old_target_loop_entry, opaque_init, _phase);
940+
}
941+
942+
// Rewire the newly created predicates to the old predicate chain head (i.e. '_current_predicate_chain_head') by
943+
// rewiring the current control input of '_current_predicate_chain_head' from '_old_target_loop_entry' to
944+
// 'initialized_assertion_predicate_success_proj'. This is required because we walk the predicate chain from the loop
945+
// up and clone Template Assertion Predicates on the fly:
946+
//
947+
// x
948+
// | old target
949+
// Template Assertion loop entry
950+
// Predicate 1 old target clone | \
951+
// | loop entry TAP 2 | cloned Template Assertion
952+
// Template Assertion | ======> | Predicate 2
953+
// Predicate 2 target loop |
954+
// | target loop #_current_predicate_chain_head
955+
// source loop
956+
//
957+
//
958+
// old target old target
959+
// loop entry loop entry
960+
// | \ rewire |
961+
// | cloned Template Assertion to old cloned Template Assertion #current_predicate
962+
// initialize | Predicate 2 predicate Predicate 2 _chain_head (new)
963+
// TAP 2 | | chain head |
964+
// ======> | Initialized Assertion ======> Initialized Assertion
965+
// | Predicate 2 Predicate 2
966+
// | |
967+
// target loop #_current_predicate_chain_head target loop
968+
//
969+
void CreateAssertionPredicatesVisitor::rewire_to_old_predicate_chain_head(
970+
Node* initialized_assertion_predicate_success_proj) const {
971+
if (_current_predicate_chain_head->is_Loop()) {
972+
assert(_current_predicate_chain_head->in(LoopNode::EntryControl) == _old_target_loop_entry, "must be old loop entry");
973+
_phase->replace_loop_entry(_current_predicate_chain_head->as_Loop(), initialized_assertion_predicate_success_proj);
974+
} else {
975+
assert(_current_predicate_chain_head->in(0) == _old_target_loop_entry, "must be old loop entry");
976+
_phase->replace_control(_current_predicate_chain_head, initialized_assertion_predicate_success_proj);
977+
}
922978
}
923979

924980
// Clone the Template Assertion Predicate and set a new input for the OpaqueLoopStrideNode.
@@ -951,9 +1007,8 @@ IfTrueNode* UpdateStrideForAssertionPredicates::initialize_from_updated_template
9511007
void UpdateStrideForAssertionPredicates::connect_initialized_assertion_predicate(
9521008
Node* new_control_out, IfTrueNode* initialized_success_proj) const {
9531009
if (new_control_out->is_Loop()) {
954-
_phase->igvn().replace_input_of(new_control_out, LoopNode::EntryControl, initialized_success_proj);
1010+
_phase->replace_loop_entry(new_control_out->as_Loop(), initialized_success_proj);
9551011
} else {
956-
_phase->igvn().replace_input_of(new_control_out, 0, initialized_success_proj);
1012+
_phase->replace_control(new_control_out, initialized_success_proj);
9571013
}
958-
_phase->set_idom(new_control_out, initialized_success_proj, _phase->dom_depth(new_control_out));
9591014
}

‎src/hotspot/share/opto/predicates.hpp

+6-26
Original file line numberDiff line numberDiff line change
@@ -984,46 +984,26 @@ class CreateAssertionPredicatesVisitor : public PredicateVisitor {
984984
Node* const _init;
985985
Node* const _stride;
986986
Node* const _old_target_loop_entry;
987-
Node* _new_control;
987+
Node* _current_predicate_chain_head;
988988
PhaseIdealLoop* const _phase;
989989
bool _has_hoisted_check_parse_predicates;
990990
const NodeInLoopBody& _node_in_loop_body;
991991
const bool _clone_template;
992992

993993
IfTrueNode* clone_template_and_replace_init_input(const TemplateAssertionPredicate& template_assertion_predicate);
994-
IfTrueNode* initialize_from_template(const TemplateAssertionPredicate& template_assertion_predicate) const;
994+
IfTrueNode* initialize_from_template(const TemplateAssertionPredicate& template_assertion_predicate,
995+
Node* new_control) const;
996+
void rewire_to_old_predicate_chain_head(Node* initialized_assertion_predicate_success_proj) const;
995997

996998
public:
997-
CreateAssertionPredicatesVisitor(Node* init, Node* stride, Node* new_control, PhaseIdealLoop* phase,
998-
const NodeInLoopBody& node_in_loop_body, const bool clone_template)
999-
: _init(init),
1000-
_stride(stride),
1001-
_old_target_loop_entry(new_control),
1002-
_new_control(new_control),
1003-
_phase(phase),
1004-
_has_hoisted_check_parse_predicates(false),
1005-
_node_in_loop_body(node_in_loop_body),
1006-
_clone_template(clone_template) {}
999+
CreateAssertionPredicatesVisitor(CountedLoopNode* target_loop_head, PhaseIdealLoop* phase,
1000+
const NodeInLoopBody& node_in_loop_body, bool clone_template);
10071001
NONCOPYABLE(CreateAssertionPredicatesVisitor);
10081002

10091003
using PredicateVisitor::visit;
10101004

10111005
void visit(const ParsePredicate& parse_predicate) override;
10121006
void visit(const TemplateAssertionPredicate& template_assertion_predicate) override;
1013-
1014-
// Did we create any new Initialized Assertion Predicates?
1015-
bool has_created_predicates() const {
1016-
return _new_control != _old_target_loop_entry;
1017-
}
1018-
1019-
// Return the last created node by this visitor or the originally provided 'new_control' to the visitor if there was
1020-
// no new node created (i.e. no Template Assertion Predicates found).
1021-
IfTrueNode* last_created_success_proj() const {
1022-
assert(has_created_predicates(), "should only be queried if new nodes have been created");
1023-
assert(_new_control->unique_ctrl_out_or_null() == nullptr, "no control outputs, yet");
1024-
assert(_new_control->is_IfTrue(), "Assertion Predicates only have IfTrue on success proj");
1025-
return _new_control->as_IfTrue();
1026-
}
10271007
};
10281008

10291009
// This visitor collects all Template Assertion Predicates If nodes or the corresponding Opaque nodes, depending on the

0 commit comments

Comments
 (0)
Please sign in to comment.