mirror of
https://github.com/openjdk/jdk.git
synced 2025-09-24 04:54:40 +02:00
8080325: SuperWord loop unrolling analysis
Determine loop unroll factor based on supported vectors sizes. Reviewed-by: roland, kvn
This commit is contained in:
Michael Berg
parent
fd10da6423
commit
7c7b91845f
7 changed files with 300 additions and 54 deletions
|
|
@ -58,4 +58,6 @@ void Compile::pd_compiler2_init() {
|
|||
OptoReg::invalidate(i);
|
||||
}
|
||||
}
|
||||
|
||||
SuperWordLoopUnrollAnalysis = true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -191,6 +191,13 @@
|
|||
product(intx, LoopMaxUnroll, 16, \
|
||||
"Maximum number of unrolls for main loop") \
|
||||
\
|
||||
product(bool, SuperWordLoopUnrollAnalysis, false, \
|
||||
"Map number of unrolls for main loop via " \
|
||||
"Superword Level Parallelism analysis") \
|
||||
\
|
||||
notproduct(bool, TraceSuperWordLoopUnrollAnalysis, false, \
|
||||
"Trace what Superword Level Parallelism analysis applies") \
|
||||
\
|
||||
product(intx, LoopUnrollMin, 4, \
|
||||
"Minimum number of unroll loop bodies before checking progress" \
|
||||
"of rounds of unroll,optimize,..") \
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@
|
|||
#include "opto/rootnode.hpp"
|
||||
#include "opto/runtime.hpp"
|
||||
#include "opto/subnode.hpp"
|
||||
#include "opto/superword.hpp"
|
||||
#include "opto/vectornode.hpp"
|
||||
|
||||
//------------------------------is_loop_exit-----------------------------------
|
||||
|
|
@ -640,7 +641,7 @@ bool IdealLoopTree::policy_maximally_unroll( PhaseIdealLoop *phase ) const {
|
|||
//------------------------------policy_unroll----------------------------------
|
||||
// Return TRUE or FALSE if the loop should be unrolled or not. Unroll if
|
||||
// the loop is a CountedLoop and the body is small enough.
|
||||
bool IdealLoopTree::policy_unroll( PhaseIdealLoop *phase ) const {
|
||||
bool IdealLoopTree::policy_unroll(PhaseIdealLoop *phase) {
|
||||
|
||||
CountedLoopNode *cl = _head->as_CountedLoop();
|
||||
assert(cl->is_normal_loop() || cl->is_main_loop(), "");
|
||||
|
|
@ -652,6 +653,8 @@ bool IdealLoopTree::policy_unroll( PhaseIdealLoop *phase ) const {
|
|||
// After split at least one iteration will be executed in pre-loop.
|
||||
if (cl->trip_count() <= (uint)(cl->is_normal_loop() ? 2 : 1)) return false;
|
||||
|
||||
_local_loop_unroll_limit = LoopUnrollLimit;
|
||||
_local_loop_unroll_factor = 4;
|
||||
int future_unroll_ct = cl->unrolled_count() * 2;
|
||||
if (future_unroll_ct > LoopMaxUnroll) return false;
|
||||
|
||||
|
|
@ -747,8 +750,24 @@ bool IdealLoopTree::policy_unroll( PhaseIdealLoop *phase ) const {
|
|||
} // switch
|
||||
}
|
||||
|
||||
if (UseSuperWord) {
|
||||
if (!cl->is_reduction_loop()) {
|
||||
phase->mark_reductions(this);
|
||||
}
|
||||
|
||||
// Only attempt slp analysis when user controls do not prohibit it
|
||||
if (LoopMaxUnroll > _local_loop_unroll_factor) {
|
||||
// Once policy_slp_analysis succeeds, mark the loop with the
|
||||
// maximal unroll factor so that we minimize analysis passes
|
||||
if ((future_unroll_ct > _local_loop_unroll_factor) ||
|
||||
(body_size > (uint)_local_loop_unroll_limit)) {
|
||||
policy_unroll_slp_analysis(cl, phase, future_unroll_ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for being too big
|
||||
if (body_size > (uint)LoopUnrollLimit) {
|
||||
if (body_size > (uint)_local_loop_unroll_limit) {
|
||||
if (xors_in_loop >= 4 && body_size < (uint)LoopUnrollLimit*4) return true;
|
||||
// Normal case: loop too big
|
||||
return false;
|
||||
|
|
@ -758,6 +777,36 @@ bool IdealLoopTree::policy_unroll( PhaseIdealLoop *phase ) const {
|
|||
return true;
|
||||
}
|
||||
|
||||
void IdealLoopTree::policy_unroll_slp_analysis(CountedLoopNode *cl, PhaseIdealLoop *phase, int future_unroll_ct) {
|
||||
// Enable this functionality target by target as needed
|
||||
if (SuperWordLoopUnrollAnalysis) {
|
||||
if (!cl->has_passed_slp()) {
|
||||
SuperWord sw(phase);
|
||||
sw.transform_loop(this, false);
|
||||
|
||||
// If the loop is slp canonical analyze it
|
||||
if (sw.early_return() == false) {
|
||||
sw.unrolling_analysis(cl, _local_loop_unroll_factor);
|
||||
}
|
||||
}
|
||||
|
||||
int slp_max_unroll_factor = cl->slp_max_unroll();
|
||||
if ((slp_max_unroll_factor > 4) &&
|
||||
(slp_max_unroll_factor >= future_unroll_ct)) {
|
||||
int new_limit = cl->node_count_before_unroll() * slp_max_unroll_factor;
|
||||
if (new_limit > LoopUnrollLimit) {
|
||||
#ifndef PRODUCT
|
||||
if (TraceSuperWordLoopUnrollAnalysis) {
|
||||
tty->print_cr("slp analysis is applying unroll limit %d, the original limit was %d\n",
|
||||
new_limit, _local_loop_unroll_limit);
|
||||
}
|
||||
#endif
|
||||
_local_loop_unroll_limit = new_limit;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//------------------------------policy_align-----------------------------------
|
||||
// Return TRUE or FALSE if the loop should be cache-line aligned. Gather the
|
||||
// expression that does the alignment. Note that only one array base can be
|
||||
|
|
@ -1611,6 +1660,7 @@ void PhaseIdealLoop::mark_reductions(IdealLoopTree *loop) {
|
|||
// iff the uses conform
|
||||
if (ok) {
|
||||
def_node->add_flag(Node::Flag_is_reduction);
|
||||
loop_head->mark_has_reductions();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2517,7 +2567,6 @@ bool IdealLoopTree::iteration_split_impl( PhaseIdealLoop *phase, Node_List &old_
|
|||
// and we'd rather unroll the post-RCE'd loop SO... do not unroll if
|
||||
// peeling.
|
||||
if (should_unroll && !should_peel) {
|
||||
phase->mark_reductions(this);
|
||||
phase->do_unroll(this, old_new, true);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2408,7 +2408,7 @@ void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts)
|
|||
for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
|
||||
IdealLoopTree* lpt = iter.current();
|
||||
if (lpt->is_counted()) {
|
||||
sw.transform_loop(lpt);
|
||||
sw.transform_loop(lpt, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,7 +62,9 @@ protected:
|
|||
HasExactTripCount=8,
|
||||
InnerLoop=16,
|
||||
PartialPeelLoop=32,
|
||||
PartialPeelFailed=64 };
|
||||
PartialPeelFailed=64,
|
||||
HasReductions=128,
|
||||
PassedSlpAnalysis=256 };
|
||||
char _unswitch_count;
|
||||
enum { _unswitch_max=3 };
|
||||
|
||||
|
|
@ -77,6 +79,8 @@ public:
|
|||
void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; }
|
||||
int partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; }
|
||||
void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; }
|
||||
void mark_has_reductions() { _loop_flags |= HasReductions; }
|
||||
void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; }
|
||||
|
||||
int unswitch_max() { return _unswitch_max; }
|
||||
int unswitch_count() { return _unswitch_count; }
|
||||
|
|
@ -155,11 +159,15 @@ class CountedLoopNode : public LoopNode {
|
|||
// unroll,optimize,unroll,optimize,... is making progress
|
||||
int _node_count_before_unroll;
|
||||
|
||||
// If slp analysis is performed we record the maximum
|
||||
// vector mapped unroll factor here
|
||||
int _slp_maximum_unroll_factor;
|
||||
|
||||
public:
|
||||
CountedLoopNode( Node *entry, Node *backedge )
|
||||
: LoopNode(entry, backedge), _main_idx(0), _trip_count(max_juint),
|
||||
_profile_trip_cnt(COUNT_UNKNOWN), _unrolled_count_log2(0),
|
||||
_node_count_before_unroll(0) {
|
||||
_node_count_before_unroll(0), _slp_maximum_unroll_factor(0) {
|
||||
init_class_id(Class_CountedLoop);
|
||||
// Initialize _trip_count to the largest possible value.
|
||||
// Will be reset (lower) if the loop's trip count is known.
|
||||
|
|
@ -199,10 +207,12 @@ public:
|
|||
|
||||
// A 'main' loop that is ONLY unrolled or peeled, never RCE'd or
|
||||
// Aligned, may be missing it's pre-loop.
|
||||
int is_normal_loop() const { return (_loop_flags&PreMainPostFlagsMask) == Normal; }
|
||||
int is_normal_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Normal; }
|
||||
int is_pre_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Pre; }
|
||||
int is_main_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Main; }
|
||||
int is_post_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Post; }
|
||||
int is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; }
|
||||
int has_passed_slp () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; }
|
||||
int is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; }
|
||||
void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; }
|
||||
|
||||
|
|
@ -234,6 +244,8 @@ public:
|
|||
|
||||
void set_node_count_before_unroll(int ct) { _node_count_before_unroll = ct; }
|
||||
int node_count_before_unroll() { return _node_count_before_unroll; }
|
||||
void set_slp_max_unroll(int unroll_factor) { _slp_maximum_unroll_factor = unroll_factor; }
|
||||
int slp_max_unroll() const { return _slp_maximum_unroll_factor; }
|
||||
|
||||
#ifndef PRODUCT
|
||||
virtual void dump_spec(outputStream *st) const;
|
||||
|
|
@ -336,6 +348,8 @@ public:
|
|||
Node *_tail; // Tail of loop
|
||||
inline Node *tail(); // Handle lazy update of _tail field
|
||||
PhaseIdealLoop* _phase;
|
||||
int _local_loop_unroll_limit;
|
||||
int _local_loop_unroll_factor;
|
||||
|
||||
Node_List _body; // Loop body for inner loops
|
||||
|
||||
|
|
@ -356,7 +370,8 @@ public:
|
|||
_safepts(NULL),
|
||||
_required_safept(NULL),
|
||||
_allow_optimizations(true),
|
||||
_nest(0), _irreducible(0), _has_call(0), _has_sfpt(0), _rce_candidate(0)
|
||||
_nest(0), _irreducible(0), _has_call(0), _has_sfpt(0), _rce_candidate(0),
|
||||
_local_loop_unroll_limit(0), _local_loop_unroll_factor(0)
|
||||
{ }
|
||||
|
||||
// Is 'l' a member of 'this'?
|
||||
|
|
@ -444,7 +459,10 @@ public:
|
|||
|
||||
// Return TRUE or FALSE if the loop should be unrolled or not. Unroll if
|
||||
// the loop is a CountedLoop and the body is small enough.
|
||||
bool policy_unroll( PhaseIdealLoop *phase ) const;
|
||||
bool policy_unroll(PhaseIdealLoop *phase);
|
||||
|
||||
// Loop analyses to map to a maximal superword unrolling for vectorization.
|
||||
void policy_unroll_slp_analysis(CountedLoopNode *cl, PhaseIdealLoop *phase, int future_unroll_ct);
|
||||
|
||||
// Return TRUE or FALSE if the loop should be range-check-eliminated.
|
||||
// Gather a list of IF tests that are dominated by iteration splitting;
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ SuperWord::SuperWord(PhaseIdealLoop* phase) :
|
|||
_bb(NULL), // basic block
|
||||
_iv(NULL), // induction var
|
||||
_race_possible(false), // cases where SDMU is true
|
||||
_early_return(true), // analysis evaluations routine
|
||||
_num_work_vecs(0), // amount of vector work we have
|
||||
_num_reductions(0), // amount of reduction work we have
|
||||
_do_vector_loop(phase->C->do_vector_loop()), // whether to do vectorization/simd style
|
||||
|
|
@ -78,7 +79,7 @@ SuperWord::SuperWord(PhaseIdealLoop* phase) :
|
|||
{}
|
||||
|
||||
//------------------------------transform_loop---------------------------
|
||||
void SuperWord::transform_loop(IdealLoopTree* lpt) {
|
||||
void SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) {
|
||||
assert(UseSuperWord, "should be");
|
||||
// Do vectors exist on this architecture?
|
||||
if (Matcher::vector_width_in_bytes(T_BYTE) < 2) return;
|
||||
|
|
@ -113,8 +114,158 @@ void SuperWord::transform_loop(IdealLoopTree* lpt) {
|
|||
// For now, define one block which is the entire loop body
|
||||
set_bb(cl);
|
||||
|
||||
if (do_optimization) {
|
||||
assert(_packset.length() == 0, "packset must be empty");
|
||||
SLP_extract();
|
||||
}
|
||||
}
|
||||
|
||||
//------------------------------early unrolling analysis------------------------------
|
||||
void SuperWord::unrolling_analysis(CountedLoopNode *cl, int &local_loop_unroll_factor) {
|
||||
bool is_slp = true;
|
||||
ResourceMark rm;
|
||||
size_t ignored_size = lpt()->_body.size();
|
||||
int *ignored_loop_nodes = NEW_RESOURCE_ARRAY(int, ignored_size);
|
||||
Node_Stack nstack((int)ignored_size);
|
||||
Node *cl_exit = cl->loopexit();
|
||||
|
||||
// First clear the entries
|
||||
for (uint i = 0; i < lpt()->_body.size(); i++) {
|
||||
ignored_loop_nodes[i] = -1;
|
||||
}
|
||||
|
||||
int max_vector = Matcher::max_vector_size(T_INT);
|
||||
|
||||
// Process the loop, some/all of the stack entries will not be in order, ergo
|
||||
// need to preprocess the ignored initial state before we process the loop
|
||||
for (uint i = 0; i < lpt()->_body.size(); i++) {
|
||||
Node* n = lpt()->_body.at(i);
|
||||
if (n == cl->incr() ||
|
||||
n->is_reduction() ||
|
||||
n->is_AddP() ||
|
||||
n->is_Cmp() ||
|
||||
n->is_IfTrue() ||
|
||||
n->is_CountedLoop() ||
|
||||
(n == cl_exit)) {
|
||||
ignored_loop_nodes[i] = n->_idx;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (n->is_If()) {
|
||||
IfNode *iff = n->as_If();
|
||||
if (iff->_fcnt != COUNT_UNKNOWN && iff->_prob != PROB_UNKNOWN) {
|
||||
if (lpt()->is_loop_exit(iff)) {
|
||||
ignored_loop_nodes[i] = n->_idx;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (n->is_Phi() && (n->bottom_type() == Type::MEMORY)) {
|
||||
Node* n_tail = n->in(LoopNode::LoopBackControl);
|
||||
if (n_tail != n->in(LoopNode::EntryControl)) {
|
||||
if (!n_tail->is_Mem()) {
|
||||
is_slp = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This must happen after check of phi/if
|
||||
if (n->is_Phi() || n->is_If()) {
|
||||
ignored_loop_nodes[i] = n->_idx;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (n->is_LoadStore() || n->is_MergeMem() ||
|
||||
(n->is_Proj() && !n->as_Proj()->is_CFG())) {
|
||||
is_slp = false;
|
||||
break;
|
||||
}
|
||||
|
||||
if (n->is_Mem()) {
|
||||
Node* adr = n->in(MemNode::Address);
|
||||
Node* n_ctrl = _phase->get_ctrl(adr);
|
||||
|
||||
// save a queue of post process nodes
|
||||
if (n_ctrl != NULL && lpt()->is_member(_phase->get_loop(n_ctrl))) {
|
||||
MemNode* current = n->as_Mem();
|
||||
BasicType bt = current->memory_type();
|
||||
if (is_java_primitive(bt) == false) {
|
||||
ignored_loop_nodes[i] = n->_idx;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Process the memory expression
|
||||
int stack_idx = 0;
|
||||
bool have_side_effects = true;
|
||||
if (adr->is_AddP() == false) {
|
||||
nstack.push(adr, stack_idx++);
|
||||
} else {
|
||||
// Mark the components of the memory operation in nstack
|
||||
SWPointer p1(current, this, &nstack, true);
|
||||
have_side_effects = p1.node_stack()->is_nonempty();
|
||||
}
|
||||
|
||||
// Process the pointer stack
|
||||
while (have_side_effects) {
|
||||
Node* pointer_node = nstack.node();
|
||||
for (uint j = 0; j < lpt()->_body.size(); j++) {
|
||||
Node* cur_node = lpt()->_body.at(j);
|
||||
if (cur_node == pointer_node) {
|
||||
ignored_loop_nodes[j] = cur_node->_idx;
|
||||
break;
|
||||
}
|
||||
}
|
||||
nstack.pop();
|
||||
have_side_effects = nstack.is_nonempty();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (is_slp) {
|
||||
// Now we try to find the maximum supported consistent vector which the machine
|
||||
// description can use
|
||||
for (uint i = 0; i < lpt()->_body.size(); i++) {
|
||||
if (ignored_loop_nodes[i] != -1) continue;
|
||||
|
||||
BasicType bt;
|
||||
Node* n = lpt()->_body.at(i);
|
||||
if (n->is_Store()) {
|
||||
bt = n->as_Mem()->memory_type();
|
||||
}
|
||||
else {
|
||||
bt = n->bottom_type()->basic_type();
|
||||
}
|
||||
|
||||
int cur_max_vector = Matcher::max_vector_size(bt);
|
||||
|
||||
// If a max vector exists which is not larger than _local_loop_unroll_factor
|
||||
// stop looking, we already have the max vector to map to.
|
||||
if (cur_max_vector <= local_loop_unroll_factor) {
|
||||
is_slp = false;
|
||||
#ifndef PRODUCT
|
||||
if (TraceSuperWordLoopUnrollAnalysis) {
|
||||
tty->print_cr("slp analysis fails: unroll limit equals max vector\n");
|
||||
}
|
||||
#endif
|
||||
break;
|
||||
}
|
||||
|
||||
// Map the maximal common vector
|
||||
if (VectorNode::implemented(n->Opcode(), cur_max_vector, bt)) {
|
||||
if (cur_max_vector < max_vector) {
|
||||
max_vector = cur_max_vector;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (is_slp) {
|
||||
local_loop_unroll_factor = max_vector;
|
||||
}
|
||||
cl->mark_passed_slp();
|
||||
cl->set_slp_max_unroll(local_loop_unroll_factor);
|
||||
}
|
||||
}
|
||||
|
||||
//------------------------------SLP_extract---------------------------
|
||||
|
|
@ -268,12 +419,12 @@ void SuperWord::find_adjacent_refs() {
|
|||
best_iv_adjustment = iv_adjustment;
|
||||
}
|
||||
|
||||
SWPointer align_to_ref_p(mem_ref, this);
|
||||
SWPointer align_to_ref_p(mem_ref, this, NULL, false);
|
||||
// Set alignment relative to "align_to_ref" for all related memory operations.
|
||||
for (int i = memops.size() - 1; i >= 0; i--) {
|
||||
MemNode* s = memops.at(i)->as_Mem();
|
||||
if (isomorphic(s, mem_ref)) {
|
||||
SWPointer p2(s, this);
|
||||
SWPointer p2(s, this, NULL, false);
|
||||
if (p2.comparable(align_to_ref_p)) {
|
||||
int align = memory_alignment(s, iv_adjustment);
|
||||
set_alignment(s, align);
|
||||
|
|
@ -294,7 +445,7 @@ void SuperWord::find_adjacent_refs() {
|
|||
// iterations in pre-loop will be not enough to align it.
|
||||
create_pack = false;
|
||||
} else {
|
||||
SWPointer p2(best_align_to_mem_ref, this);
|
||||
SWPointer p2(best_align_to_mem_ref, this, NULL, false);
|
||||
if (align_to_ref_p.invar() != p2.invar()) {
|
||||
// Do not vectorize memory accesses with different invariants
|
||||
// if unaligned memory accesses are not allowed.
|
||||
|
|
@ -411,7 +562,7 @@ MemNode* SuperWord::find_align_to_ref(Node_List &memops) {
|
|||
// Count number of comparable memory ops
|
||||
for (uint i = 0; i < memops.size(); i++) {
|
||||
MemNode* s1 = memops.at(i)->as_Mem();
|
||||
SWPointer p1(s1, this);
|
||||
SWPointer p1(s1, this, NULL, false);
|
||||
// Discard if pre loop can't align this reference
|
||||
if (!ref_is_alignable(p1)) {
|
||||
*cmp_ct.adr_at(i) = 0;
|
||||
|
|
@ -420,7 +571,7 @@ MemNode* SuperWord::find_align_to_ref(Node_List &memops) {
|
|||
for (uint j = i+1; j < memops.size(); j++) {
|
||||
MemNode* s2 = memops.at(j)->as_Mem();
|
||||
if (isomorphic(s1, s2)) {
|
||||
SWPointer p2(s2, this);
|
||||
SWPointer p2(s2, this, NULL, false);
|
||||
if (p1.comparable(p2)) {
|
||||
(*cmp_ct.adr_at(i))++;
|
||||
(*cmp_ct.adr_at(j))++;
|
||||
|
|
@ -441,7 +592,7 @@ MemNode* SuperWord::find_align_to_ref(Node_List &memops) {
|
|||
if (s->is_Store()) {
|
||||
int vw = vector_width_in_bytes(s);
|
||||
assert(vw > 1, "sanity");
|
||||
SWPointer p(s, this);
|
||||
SWPointer p(s, this, NULL, false);
|
||||
if (cmp_ct.at(j) > max_ct ||
|
||||
cmp_ct.at(j) == max_ct &&
|
||||
(vw > max_vw ||
|
||||
|
|
@ -464,7 +615,7 @@ MemNode* SuperWord::find_align_to_ref(Node_List &memops) {
|
|||
if (s->is_Load()) {
|
||||
int vw = vector_width_in_bytes(s);
|
||||
assert(vw > 1, "sanity");
|
||||
SWPointer p(s, this);
|
||||
SWPointer p(s, this, NULL, false);
|
||||
if (cmp_ct.at(j) > max_ct ||
|
||||
cmp_ct.at(j) == max_ct &&
|
||||
(vw > max_vw ||
|
||||
|
|
@ -575,7 +726,7 @@ bool SuperWord::ref_is_alignable(SWPointer& p) {
|
|||
//---------------------------get_iv_adjustment---------------------------
|
||||
// Calculate loop's iv adjustment for this memory ops.
|
||||
int SuperWord::get_iv_adjustment(MemNode* mem_ref) {
|
||||
SWPointer align_to_ref_p(mem_ref, this);
|
||||
SWPointer align_to_ref_p(mem_ref, this, NULL, false);
|
||||
int offset = align_to_ref_p.offset_in_bytes();
|
||||
int scale = align_to_ref_p.scale_in_bytes();
|
||||
int elt_size = align_to_ref_p.memory_size();
|
||||
|
|
@ -649,13 +800,13 @@ void SuperWord::dependence_graph() {
|
|||
if (_dg.dep(s1)->in_cnt() == 0) {
|
||||
_dg.make_edge(slice, s1);
|
||||
}
|
||||
SWPointer p1(s1->as_Mem(), this);
|
||||
SWPointer p1(s1->as_Mem(), this, NULL, false);
|
||||
bool sink_dependent = true;
|
||||
for (int k = j - 1; k >= 0; k--) {
|
||||
Node* s2 = _nlist.at(k);
|
||||
if (s1->is_Load() && s2->is_Load())
|
||||
continue;
|
||||
SWPointer p2(s2->as_Mem(), this);
|
||||
SWPointer p2(s2->as_Mem(), this, NULL, false);
|
||||
|
||||
int cmp = p1.cmp(p2);
|
||||
if (SuperWordRTDepCheck &&
|
||||
|
|
@ -795,8 +946,8 @@ bool SuperWord::are_adjacent_refs(Node* s1, Node* s2) {
|
|||
if (_phase->C->get_alias_index(s1->as_Mem()->adr_type()) !=
|
||||
_phase->C->get_alias_index(s2->as_Mem()->adr_type()))
|
||||
return false;
|
||||
SWPointer p1(s1->as_Mem(), this);
|
||||
SWPointer p2(s2->as_Mem(), this);
|
||||
SWPointer p1(s1->as_Mem(), this, NULL, false);
|
||||
SWPointer p2(s2->as_Mem(), this, NULL, false);
|
||||
if (p1.base() != p2.base() || !p1.comparable(p2)) return false;
|
||||
int diff = p2.offset_in_bytes() - p1.offset_in_bytes();
|
||||
return diff == data_size(s1);
|
||||
|
|
@ -1615,13 +1766,13 @@ void SuperWord::output() {
|
|||
if (n->is_Load()) {
|
||||
Node* ctl = n->in(MemNode::Control);
|
||||
Node* mem = first->in(MemNode::Memory);
|
||||
SWPointer p1(n->as_Mem(), this);
|
||||
SWPointer p1(n->as_Mem(), this, NULL, false);
|
||||
// Identify the memory dependency for the new loadVector node by
|
||||
// walking up through memory chain.
|
||||
// This is done to give flexibility to the new loadVector node so that
|
||||
// it can move above independent storeVector nodes.
|
||||
while (mem->is_StoreVector()) {
|
||||
SWPointer p2(mem->as_Mem(), this);
|
||||
SWPointer p2(mem->as_Mem(), this, NULL, false);
|
||||
int cmp = p1.cmp(p2);
|
||||
if (SWPointer::not_equal(cmp) || !SWPointer::comparable(cmp)) {
|
||||
mem = mem->in(MemNode::Memory);
|
||||
|
|
@ -2138,7 +2289,7 @@ void SuperWord::compute_vector_element_type() {
|
|||
//------------------------------memory_alignment---------------------------
|
||||
// Alignment within a vector memory reference
|
||||
int SuperWord::memory_alignment(MemNode* s, int iv_adjust) {
|
||||
SWPointer p(s, this);
|
||||
SWPointer p(s, this, NULL, false);
|
||||
if (!p.valid()) {
|
||||
return bottom_align;
|
||||
}
|
||||
|
|
@ -2315,7 +2466,7 @@ void SuperWord::align_initial_loop_index(MemNode* align_to_ref) {
|
|||
Node *orig_limit = pre_opaq->original_loop_limit();
|
||||
assert(orig_limit != NULL && _igvn.type(orig_limit) != Type::TOP, "");
|
||||
|
||||
SWPointer align_to_ref_p(align_to_ref, this);
|
||||
SWPointer align_to_ref_p(align_to_ref, this, NULL, false);
|
||||
assert(align_to_ref_p.valid(), "sanity");
|
||||
|
||||
// Given:
|
||||
|
|
@ -2489,6 +2640,7 @@ void SuperWord::init() {
|
|||
_bb = NULL;
|
||||
_iv = NULL;
|
||||
_race_possible = 0;
|
||||
_early_return = false;
|
||||
_num_work_vecs = 0;
|
||||
_num_reductions = 0;
|
||||
}
|
||||
|
|
@ -2559,9 +2711,11 @@ char* SuperWord::blank(uint depth) {
|
|||
//==============================SWPointer===========================
|
||||
|
||||
//----------------------------SWPointer------------------------
|
||||
SWPointer::SWPointer(MemNode* mem, SuperWord* slp) :
|
||||
SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool analyze_only) :
|
||||
_mem(mem), _slp(slp), _base(NULL), _adr(NULL),
|
||||
_scale(0), _offset(0), _invar(NULL), _negate_invar(false) {
|
||||
_scale(0), _offset(0), _invar(NULL), _negate_invar(false),
|
||||
_nstack(nstack), _analyze_only(analyze_only),
|
||||
_stack_idx(0) {
|
||||
|
||||
Node* adr = mem->in(MemNode::Address);
|
||||
if (!adr->is_AddP()) {
|
||||
|
|
@ -2599,7 +2753,9 @@ SWPointer::SWPointer(MemNode* mem, SuperWord* slp) :
|
|||
// the pattern match of an address expression.
|
||||
SWPointer::SWPointer(SWPointer* p) :
|
||||
_mem(p->_mem), _slp(p->_slp), _base(NULL), _adr(NULL),
|
||||
_scale(0), _offset(0), _invar(NULL), _negate_invar(false) {}
|
||||
_scale(0), _offset(0), _invar(NULL), _negate_invar(false),
|
||||
_nstack(p->_nstack), _analyze_only(p->_analyze_only),
|
||||
_stack_idx(p->_stack_idx) {}
|
||||
|
||||
//------------------------scaled_iv_plus_offset--------------------
|
||||
// Match: k*iv + offset
|
||||
|
|
@ -2642,6 +2798,9 @@ bool SWPointer::scaled_iv(Node* n) {
|
|||
_scale = 1;
|
||||
return true;
|
||||
}
|
||||
if (_analyze_only && (invariant(n) == false)) {
|
||||
_nstack->push(n, _stack_idx++);
|
||||
}
|
||||
int opc = n->Opcode();
|
||||
if (opc == Op_MulI) {
|
||||
if (n->in(1) == iv() && n->in(2)->is_Con()) {
|
||||
|
|
@ -2699,6 +2858,9 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) {
|
|||
return false;
|
||||
}
|
||||
if (_invar != NULL) return false; // already have an invariant
|
||||
if (_analyze_only && (invariant(n) == false)) {
|
||||
_nstack->push(n, _stack_idx++);
|
||||
}
|
||||
if (opc == Op_AddI) {
|
||||
if (n->in(2)->is_Con() && invariant(n->in(1))) {
|
||||
_negate_invar = negate;
|
||||
|
|
|
|||
|
|
@ -239,12 +239,15 @@ class SuperWord : public ResourceObj {
|
|||
public:
|
||||
SuperWord(PhaseIdealLoop* phase);
|
||||
|
||||
void transform_loop(IdealLoopTree* lpt);
|
||||
void transform_loop(IdealLoopTree* lpt, bool do_optimization);
|
||||
|
||||
void unrolling_analysis(CountedLoopNode *cl, int &local_loop_unroll_factor);
|
||||
|
||||
// Accessors for SWPointer
|
||||
PhaseIdealLoop* phase() { return _phase; }
|
||||
IdealLoopTree* lpt() { return _lpt; }
|
||||
PhiNode* iv() { return _iv; }
|
||||
bool early_return() { return _early_return; }
|
||||
|
||||
private:
|
||||
IdealLoopTree* _lpt; // Current loop tree node
|
||||
|
|
@ -252,6 +255,7 @@ class SuperWord : public ResourceObj {
|
|||
Node* _bb; // Current basic block
|
||||
PhiNode* _iv; // Induction var
|
||||
bool _race_possible; // In cases where SDMU is true
|
||||
bool _early_return; // True if we do not initialize
|
||||
bool _do_vector_loop; // whether to do vectorization/simd style
|
||||
bool _vector_loop_debug; // provide more printing in debug mode
|
||||
int _num_work_vecs; // Number of non memory vector operations
|
||||
|
|
@ -471,6 +475,9 @@ class SWPointer VALUE_OBJ_CLASS_SPEC {
|
|||
jint _offset; // constant offset (in bytes)
|
||||
Node* _invar; // invariant offset (in bytes), NULL if none
|
||||
bool _negate_invar; // if true then use: (0 - _invar)
|
||||
Node_Stack* _nstack; // stack used to record a swpointer trace of variants
|
||||
bool _analyze_only; // Used in loop unrolling only for swpointer trace
|
||||
uint _stack_idx; // Used in loop unrolling only for swpointer trace
|
||||
|
||||
PhaseIdealLoop* phase() { return _slp->phase(); }
|
||||
IdealLoopTree* lpt() { return _slp->lpt(); }
|
||||
|
|
@ -497,7 +504,7 @@ class SWPointer VALUE_OBJ_CLASS_SPEC {
|
|||
NotComparable = (Less | Greater | Equal)
|
||||
};
|
||||
|
||||
SWPointer(MemNode* mem, SuperWord* slp);
|
||||
SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool analyze_only);
|
||||
// Following is used to create a temporary object during
|
||||
// the pattern match of an address expression.
|
||||
SWPointer(SWPointer* p);
|
||||
|
|
@ -513,6 +520,7 @@ class SWPointer VALUE_OBJ_CLASS_SPEC {
|
|||
bool negate_invar() { return _negate_invar; }
|
||||
int offset_in_bytes() { return _offset; }
|
||||
int memory_size() { return _mem->memory_size(); }
|
||||
Node_Stack* node_stack() { return _nstack; }
|
||||
|
||||
// Comparable?
|
||||
int cmp(SWPointer& q) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue