mirror of
https://github.com/openjdk/jdk.git
synced 2025-09-18 01:54:47 +02:00
8248830: C2: Optimize Rotate API on x86
Improved existing scalar rotate operations, added support for vector rotate operations using new AVX512 instructions. Reviewed-by: vlivanov, kvn
This commit is contained in:
Jatin Bhateja
parent
084e15bca3
commit
3ed56830b4
19 changed files with 1474 additions and 282 deletions
|
|
@ -24,8 +24,11 @@
|
|||
#include "precompiled.hpp"
|
||||
#include "memory/allocation.inline.hpp"
|
||||
#include "opto/connode.hpp"
|
||||
#include "opto/mulnode.hpp"
|
||||
#include "opto/subnode.hpp"
|
||||
#include "opto/vectornode.hpp"
|
||||
#include "utilities/powerOfTwo.hpp"
|
||||
#include "utilities/globalDefinitions.hpp"
|
||||
|
||||
//------------------------------VectorNode--------------------------------------
|
||||
|
||||
|
|
@ -132,6 +135,12 @@ int VectorNode::opcode(int sopc, BasicType bt) {
|
|||
case Op_RoundDoubleMode:
|
||||
assert(bt == T_DOUBLE, "must be");
|
||||
return Op_RoundDoubleModeV;
|
||||
case Op_RotateLeft:
|
||||
assert(bt == T_LONG || bt == T_INT, "must be");
|
||||
return Op_RotateLeftV;
|
||||
case Op_RotateRight:
|
||||
assert(bt == T_LONG || bt == T_INT, "must be");
|
||||
return Op_RotateRightV;
|
||||
case Op_SqrtF:
|
||||
assert(bt == T_FLOAT, "must be");
|
||||
return Op_SqrtVF;
|
||||
|
|
@ -239,6 +248,12 @@ bool VectorNode::implemented(int opc, uint vlen, BasicType bt) {
|
|||
(vlen > 1) && is_power_of_2(vlen) &&
|
||||
Matcher::vector_size_supported(bt, vlen)) {
|
||||
int vopc = VectorNode::opcode(opc, bt);
|
||||
// For rotate operation we will do a lazy de-generation into
|
||||
// OrV/LShiftV/URShiftV pattern if the target does not support
|
||||
// vector rotation instruction.
|
||||
if (vopc == Op_RotateLeftV || vopc == Op_RotateRightV) {
|
||||
return is_vector_rotate_supported(vopc, vlen, bt);
|
||||
}
|
||||
return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt);
|
||||
}
|
||||
return false;
|
||||
|
|
@ -263,13 +278,45 @@ bool VectorNode::is_muladds2i(Node* n) {
|
|||
return false;
|
||||
}
|
||||
|
||||
bool VectorNode::is_roundopD(Node *n) {
|
||||
bool VectorNode::is_roundopD(Node* n) {
|
||||
if (n->Opcode() == Op_RoundDoubleMode) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool VectorNode::is_scalar_rotate(Node* n) {
|
||||
if (n->Opcode() == Op_RotateLeft || n->Opcode() == Op_RotateRight) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) {
|
||||
assert(vopc == Op_RotateLeftV || vopc == Op_RotateRightV, "wrong opcode");
|
||||
|
||||
// If target defines vector rotation patterns then no
|
||||
// need for degeneration.
|
||||
if (Matcher::match_rule_supported_vector(vopc, vlen, bt)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Validate existence of nodes created in case of rotate degeneration.
|
||||
switch (bt) {
|
||||
case T_INT:
|
||||
return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) &&
|
||||
Matcher::match_rule_supported_vector(Op_LShiftVI, vlen, bt) &&
|
||||
Matcher::match_rule_supported_vector(Op_URShiftVI, vlen, bt);
|
||||
case T_LONG:
|
||||
return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) &&
|
||||
Matcher::match_rule_supported_vector(Op_LShiftVL, vlen, bt) &&
|
||||
Matcher::match_rule_supported_vector(Op_URShiftVL, vlen, bt);
|
||||
default:
|
||||
assert(false, "not supported: %s", type2name(bt));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool VectorNode::is_shift(Node* n) {
|
||||
switch (n->Opcode()) {
|
||||
case Op_LShiftI:
|
||||
|
|
@ -395,6 +442,8 @@ VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType b
|
|||
case Op_SqrtVD: return new SqrtVDNode(n1, vt);
|
||||
|
||||
case Op_PopCountVI: return new PopCountVINode(n1, vt);
|
||||
case Op_RotateLeftV: return new RotateLeftVNode(n1, n2, vt);
|
||||
case Op_RotateRightV: return new RotateRightVNode(n1, n2, vt);
|
||||
|
||||
case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt);
|
||||
case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt);
|
||||
|
|
@ -784,3 +833,161 @@ MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node
|
|||
return new MacroLogicVNode(in1, in2, in3, fn, vt);
|
||||
}
|
||||
|
||||
Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left,
|
||||
int vlen, BasicType bt, PhaseGVN* phase) {
|
||||
int shiftLOpc;
|
||||
int shiftROpc;
|
||||
Node* shiftLCnt = NULL;
|
||||
Node* shiftRCnt = NULL;
|
||||
const TypeVect* vt = TypeVect::make(bt, vlen);
|
||||
|
||||
// Compute shift values for right rotation and
|
||||
// later swap them in case of left rotation.
|
||||
if (cnt->is_Con()) {
|
||||
// Constant shift case.
|
||||
if (bt == T_INT) {
|
||||
int shift = cnt->get_int() & 31;
|
||||
shiftRCnt = phase->intcon(shift);
|
||||
shiftLCnt = phase->intcon(32 - shift);
|
||||
shiftLOpc = Op_LShiftI;
|
||||
shiftROpc = Op_URShiftI;
|
||||
} else {
|
||||
int shift = cnt->get_int() & 63;
|
||||
shiftRCnt = phase->intcon(shift);
|
||||
shiftLCnt = phase->intcon(64 - shift);
|
||||
shiftLOpc = Op_LShiftL;
|
||||
shiftROpc = Op_URShiftL;
|
||||
}
|
||||
} else {
|
||||
// Variable shift case.
|
||||
assert(VectorNode::is_invariant_vector(cnt), "Broadcast expected");
|
||||
cnt = cnt->in(1);
|
||||
if (bt == T_INT) {
|
||||
shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(31)));
|
||||
shiftLCnt = phase->transform(new SubINode(phase->intcon(32), shiftRCnt));
|
||||
shiftLOpc = Op_LShiftI;
|
||||
shiftROpc = Op_URShiftI;
|
||||
} else {
|
||||
assert(cnt->Opcode() == Op_ConvI2L, "ConvI2L expected");
|
||||
cnt = cnt->in(1);
|
||||
shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(63)));
|
||||
shiftLCnt = phase->transform(new SubINode(phase->intcon(64), shiftRCnt));
|
||||
shiftLOpc = Op_LShiftL;
|
||||
shiftROpc = Op_URShiftL;
|
||||
}
|
||||
}
|
||||
|
||||
// Swap the computed left and right shift counts.
|
||||
if (is_rotate_left) {
|
||||
swap(shiftRCnt,shiftLCnt);
|
||||
}
|
||||
|
||||
shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
|
||||
shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
|
||||
|
||||
return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt)),
|
||||
phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt)),
|
||||
vt);
|
||||
}
|
||||
|
||||
Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
|
||||
int vlen = length();
|
||||
BasicType bt = vect_type()->element_basic_type();
|
||||
if (!Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) {
|
||||
return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
|
||||
int vlen = length();
|
||||
BasicType bt = vect_type()->element_basic_type();
|
||||
if (!Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) {
|
||||
return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
Node* OrVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
|
||||
int lopcode = in(1)->Opcode();
|
||||
int ropcode = in(2)->Opcode();
|
||||
const TypeVect* vt = bottom_type()->is_vect();
|
||||
int vec_len = vt->length();
|
||||
BasicType bt = vt->element_basic_type();
|
||||
|
||||
// Vector Rotate operations inferencing, this will be useful when vector
|
||||
// operations are created via non-SLP route i.e. (VectorAPI).
|
||||
if (Matcher::match_rule_supported_vector(Op_RotateLeftV, vec_len, bt) &&
|
||||
((ropcode == Op_LShiftVI && lopcode == Op_URShiftVI) ||
|
||||
(ropcode == Op_LShiftVL && lopcode == Op_URShiftVL)) &&
|
||||
in(1)->in(1) == in(2)->in(1)) {
|
||||
assert(Op_RShiftCntV == in(1)->in(2)->Opcode(), "LShiftCntV operand expected");
|
||||
assert(Op_LShiftCntV == in(2)->in(2)->Opcode(), "RShiftCntV operand expected");
|
||||
Node* lshift = in(1)->in(2)->in(1);
|
||||
Node* rshift = in(2)->in(2)->in(1);
|
||||
int mod_val = bt == T_LONG ? 64 : 32;
|
||||
int shift_mask = bt == T_LONG ? 0x3F : 0x1F;
|
||||
// val >> norm_con_shift | val << (32 - norm_con_shift) => rotate_right val ,
|
||||
// norm_con_shift
|
||||
if (lshift->is_Con() && rshift->is_Con() &&
|
||||
((lshift->get_int() & shift_mask) ==
|
||||
(mod_val - (rshift->get_int() & shift_mask)))) {
|
||||
return new RotateRightVNode(
|
||||
in(1)->in(1), phase->intcon(lshift->get_int() & shift_mask), vt);
|
||||
}
|
||||
if (lshift->Opcode() == Op_AndI && rshift->Opcode() == Op_AndI &&
|
||||
lshift->in(2)->is_Con() && rshift->in(2)->is_Con() &&
|
||||
lshift->in(2)->get_int() == (mod_val - 1) &&
|
||||
rshift->in(2)->get_int() == (mod_val - 1)) {
|
||||
lshift = lshift->in(1);
|
||||
rshift = rshift->in(1);
|
||||
// val << var_shift | val >> (0/32 - var_shift) => rotate_left val ,
|
||||
// var_shift
|
||||
if (lshift->Opcode() == Op_SubI && lshift->in(2) == rshift &&
|
||||
lshift->in(1)->is_Con() &&
|
||||
(lshift->in(1)->get_int() == 0 ||
|
||||
lshift->in(1)->get_int() == mod_val)) {
|
||||
Node* rotate_cnt = phase->transform(new ReplicateINode(rshift, vt));
|
||||
return new RotateLeftVNode(in(1)->in(1), rotate_cnt, vt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Matcher::match_rule_supported_vector(Op_RotateRightV, vec_len, bt) &&
|
||||
((ropcode == Op_URShiftVI && lopcode == Op_LShiftVI) ||
|
||||
(ropcode == Op_URShiftVL && lopcode == Op_LShiftVL)) &&
|
||||
in(1)->in(1) == in(2)->in(1)) {
|
||||
assert(Op_LShiftCntV == in(1)->in(2)->Opcode(), "RShiftCntV operand expected");
|
||||
assert(Op_RShiftCntV == in(2)->in(2)->Opcode(), "LShiftCntV operand expected");
|
||||
Node* rshift = in(1)->in(2)->in(1);
|
||||
Node* lshift = in(2)->in(2)->in(1);
|
||||
int mod_val = bt == T_LONG ? 64 : 32;
|
||||
int shift_mask = bt == T_LONG ? 0x3F : 0x1F;
|
||||
// val << norm_con_shift | val >> (32 - norm_con_shift) => rotate_left val
|
||||
// , norm_con_shift
|
||||
if (rshift->is_Con() && lshift->is_Con() &&
|
||||
((rshift->get_int() & shift_mask) ==
|
||||
(mod_val - (lshift->get_int() & shift_mask)))) {
|
||||
return new RotateLeftVNode(
|
||||
in(1)->in(1), phase->intcon(rshift->get_int() & shift_mask), vt);
|
||||
}
|
||||
if (lshift->Opcode() == Op_AndI && rshift->Opcode() == Op_AndI &&
|
||||
lshift->in(2)->is_Con() && rshift->in(2)->is_Con() &&
|
||||
rshift->in(2)->get_int() == (mod_val - 1) &&
|
||||
lshift->in(2)->get_int() == (mod_val - 1)) {
|
||||
rshift = rshift->in(1);
|
||||
lshift = lshift->in(1);
|
||||
// val >> var_shift | val << (0/32 - var_shift) => rotate_right val ,
|
||||
// var_shift
|
||||
if (rshift->Opcode() == Op_SubI && rshift->in(2) == lshift &&
|
||||
rshift->in(1)->is_Con() &&
|
||||
(rshift->in(1)->get_int() == 0 ||
|
||||
rshift->in(1)->get_int() == mod_val)) {
|
||||
Node* rotate_cnt = phase->transform(new ReplicateINode(lshift, vt));
|
||||
return new RotateRightVNode(in(1)->in(1), rotate_cnt, vt);
|
||||
}
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue