Go to the documentation of this file.
19 #define DEBUG_TYPE "ve-lower"
32 unsigned Opc =
Op.getOpcode();
40 const unsigned Opcode =
Op->getOpcode();
44 unsigned VVPOpcode = VVPOpcodeOpt.getValue();
52 case VEISD::VVP_STORE:
54 case VEISD::VVP_GATHER:
55 case VEISD::VVP_SCATTER:
71 Mask =
Op->getOperand(*MaskIdx);
73 AVL =
Op->getOperand(*AVLIdx);
84 return CDAG.
getNode(VVPOpcode, LegalVecVT, {
Op->getOperand(0),
Mask, AVL});
86 return CDAG.
getNode(VVPOpcode, LegalVecVT,
87 {
Op->getOperand(0),
Op->getOperand(1),
Mask, AVL});
91 SDValue VectorV =
Op->getOperand(SrcHasStart ? 1 : 0);
93 VectorV,
Mask, AVL,
Op->getFlags());
99 case VEISD::VVP_FFMA: {
102 auto X =
Op->getOperand(2);
103 auto Y =
Op->getOperand(0);
104 auto Z =
Op->getOperand(1);
105 return CDAG.
getNode(VVPOpcode, LegalVecVT, {
X,
Y, Z,
Mask, AVL});
107 case VEISD::VVP_SELECT: {
108 auto Mask =
Op->getOperand(0);
109 auto OnTrue =
Op->getOperand(1);
110 auto OnFalse =
Op->getOperand(2);
111 return CDAG.
getNode(VVPOpcode, LegalVecVT, {OnTrue, OnFalse,
Mask, AVL});
113 case VEISD::VVP_SETCC: {
115 auto LHS =
Op->getOperand(0);
116 auto RHS =
Op->getOperand(1);
117 auto Pred =
Op->getOperand(2);
126 const bool IsLoad = (VVPOpc == VEISD::VVP_LOAD);
153 Packing, DataVT.getVectorElementType().getSimpleVT());
156 {Chain, BasePtr, StrideV,
Mask, AVL});
158 if (!PassThru || PassThru->
isUndef())
163 {NewLoadV, PassThru,
Mask, AVL});
171 assert(VVPOpc == VEISD::VVP_STORE);
172 return CDAG.
getNode(VEISD::VVP_STORE,
Op.getNode()->getVTList(),
173 {Chain, Data, BasePtr, StrideV, Mask, AVL});
179 assert((VVPOC == VEISD::VVP_LOAD) || (VVPOC == VEISD::VVP_STORE));
183 "Can only split packed load/store");
187 "Should have been folded in lowering to VVP layer");
196 unsigned ChainResIdx = PackData ? 0 : 1;
208 UpperPartAVL = SplitTM.AVL;
219 CDAG.
getUnpack(SplitDataVT, PackData, Part, SplitTM.AVL);
220 OpVec.push_back(PartData);
231 OpVec.push_back(SplitTM.Mask);
232 OpVec.push_back(SplitTM.AVL);
260 return CDAG.getMergeValues({PackedVals, FusedChains});
278 if (PassThru && PassThru->
isUndef())
281 bool IsScatter = (bool)StoredValue;
295 {Chain, StoredValue, AddressVec,
Mask, AVL});
299 {Chain, AddressVec,
Mask, AVL});
306 {NewLoadV, PassThru,
Mask, AVL});
329 switch (
Op->getOpcode()) {
330 case VEISD::VVP_LOAD:
331 case VEISD::VVP_STORE:
335 EVT IdiomVT =
Op.getValueType();
352 SDValue PackedAVL = AVLPair.first;
353 assert(!AVLPair.second &&
"Expecting non pack-legalized oepration");
365 UpperPartAVL = SplitTM.AVL;
369 for (
unsigned i = 0;
i <
Op.getNumOperands(); ++
i) {
370 if (AVLPos && ((
int)
i) == *AVLPos)
372 if (MaskPos && ((
int)
i) == *MaskPos)
376 auto PackedOperand =
Op.getOperand(
i);
377 auto UnpackedOpVT =
splitVectorType(PackedOperand.getSimpleValueType());
379 CDAG.
getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL);
380 OpVec.push_back(PartV);
384 OpVec.push_back(SplitTM.Mask);
385 OpVec.push_back(SplitTM.AVL);
388 CDAG.
getNode(
Op.getOpcode(), ResVT, OpVec,
Op->getFlags());
413 "TODO Shift predication from EVL into Mask");
415 if (
auto *ConstAVL = dyn_cast<ConstantSDNode>(AVL)) {
427 int NumOp =
Op->getNumOperands();
429 std::vector<SDValue> FixedOperands;
430 for (
int i = 0;
i < NumOp; ++
i) {
431 if (AVLPos && (
i == *AVLPos)) {
432 FixedOperands.push_back(AnnotatedLegalAVL);
435 FixedOperands.push_back(
Op->getOperand(
i));
439 auto Flags =
Op->getFlags();
441 CDAG.
getNode(
Op->getOpcode(),
Op->getVTList(), FixedOperands, Flags);
SDValue legalizeInternalVectorOp(SDValue Op, SelectionDAG &DAG) const
This is an optimization pass for GlobalISel generic memory operations.
SDValue getNodeChain(SDValue Op)
Node Properties {.
SDValue lowerVVP_LOAD_STORE(SDValue Op, VECustomDAG &) const
SDValue splitVectorOp(SDValue Op, VECustomDAG &CDAG) const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNodeMask(SDValue Op)
SDValue legalizePackedAVL(SDValue Op, VECustomDAG &CDAG) const
bool isUndef() const
Return true if the type of the node type undefined.
SDValue getMergeValues(ArrayRef< SDValue > Values) const
} Packing
bool isPackedVectorType(EVT SomeVT)
Optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MVT splitVectorType(MVT VT)
Packing getTypePacking(EVT VT)
SDValue getPack(EVT DestVT, SDValue LoVec, SDValue HiVec, SDValue AVL) const
Optional< int > getMaskPos(unsigned Opc)
LLVMContext * getContext() const
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getStoredValue(SDValue Op)
SDValue splitMaskArithmetic(SDValue Op, SelectionDAG &DAG) const
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Clang compiles this i1 i64 store i64 i64 store i64 i64 store i64 i64 store i64 align Which gets codegen d xmm0 movaps rbp movaps rbp movaps rbp movaps rbp rbp rbp rbp rbp It would be better to have movq s of instead of the movaps s LLVM produces ret int
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
SDValue getGatherScatterAddress(SDValue BasePtr, SDValue Scale, SDValue Index, SDValue Mask, SDValue AVL) const
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
SDValue getNodeAVL(SDValue Op)
} Node Properties
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
std::pair< SDValue, bool > getAnnotatedNodeAVL(SDValue Op)
SDValue getConstantMask(Packing Packing, bool AllTrue) const
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool supportsPackedMode(unsigned Opcode, EVT IdiomVT)
SDValue annotateLegalAVL(SDValue AVL) const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SDValue getUnpack(EVT DestVT, SDValue Vec, PackElem Part, SDValue AVL) const
} Legalizing getNode
SDValue getLegalReductionOpVVP(unsigned VVPOpcode, EVT ResVT, SDValue StartV, SDValue VectorV, SDValue Mask, SDValue AVL, SDNodeFlags Flags) const
} getNode
SDValue getSplitPtrOffset(SDValue Ptr, SDValue ByteStride, PackElem Part) const
Optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Optional< EVT > getIdiomaticVectorType(SDNode *Op)
} AVL Functions
bool isLegalAVL(SDValue AVL)
bool isVVPOrVEC(unsigned Opcode)
bool maySafelyIgnoreMask(SDValue Op)
SDValue getGatherScatterIndex(SDValue Op)
bool isVVPBinaryOp(unsigned VVPOpcode)
bool isVVPUnaryOp(unsigned VVPOpcode)
SDValue getNodePassthru(SDValue Op)
SDValue getConstant(uint64_t Val, EVT VT, bool IsTarget=false, bool IsOpaque=false) const
SDValue legalizeInternalLoadStoreOp(SDValue Op, VECustomDAG &CDAG) const
SDValue lowerVVP_GATHER_SCATTER(SDValue Op, VECustomDAG &) const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Optional< unsigned > getVVPOpcode(unsigned Opcode)
@ ADD
Simple integer binary arithmetic operators.
bool isVVPReductionOp(unsigned Opcode)
EVT getVectorElementType() const
Given a vector type, return the type of each element.
SDValue getSplitPtrStride(SDValue PackStride) const
SDValue getNode(unsigned OC, SDVTList VTL, ArrayRef< SDValue > OpV, Optional< SDNodeFlags > Flags=None) const
getNode {
SDValue splitPackedLoadStore(SDValue Op, VECustomDAG &CDAG) const
SDValue getGatherScatterScale(SDValue Op)
Optional< int > getAVLPos(unsigned Opc)
The VE backend uses a two-staged process to lower and legalize vector instructions:
MVT getLegalVectorType(Packing P, MVT ElemVT)
SDValue getLoadStoreStride(SDValue Op, VECustomDAG &CDAG)
SDValue lowerToVVP(SDValue Op, SelectionDAG &DAG) const
} Custom Inserter
bool hasReductionStartParam(unsigned OPC)
SDValue getMemoryPtr(SDValue Op)
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
VETargetMasks getTargetSplitMask(SDValue RawMask, SDValue RawAVL, PackElem Part) const