19#define DEBUG_TYPE "ve-lower"
32 unsigned Opc = Op.getOpcode();
40 const unsigned Opcode = Op->getOpcode();
44 unsigned VVPOpcode = *VVPOpcodeOpt;
52 case VEISD::VVP_STORE:
54 case VEISD::VVP_GATHER:
55 case VEISD::VVP_SCATTER:
71 Mask = Op->getOperand(*MaskIdx);
73 AVL = Op->getOperand(*AVLIdx);
84 return CDAG.
getNode(VVPOpcode, LegalVecVT, {Op->getOperand(0), Mask, AVL});
86 return CDAG.
getNode(VVPOpcode, LegalVecVT,
87 {Op->getOperand(0), Op->getOperand(1), Mask, AVL});
91 SDValue VectorV = Op->getOperand(SrcHasStart ? 1 : 0);
93 VectorV, Mask, AVL, Op->getFlags());
99 case VEISD::VVP_FFMA: {
102 auto X = Op->getOperand(2);
103 auto Y = Op->getOperand(0);
104 auto Z = Op->getOperand(1);
105 return CDAG.
getNode(VVPOpcode, LegalVecVT, {
X,
Y, Z, Mask, AVL});
107 case VEISD::VVP_SELECT: {
108 auto Mask = Op->getOperand(0);
109 auto OnTrue = Op->getOperand(1);
110 auto OnFalse = Op->getOperand(2);
111 return CDAG.
getNode(VVPOpcode, LegalVecVT, {OnTrue, OnFalse, Mask, AVL});
113 case VEISD::VVP_SETCC: {
115 auto LHS = Op->getOperand(0);
116 auto RHS = Op->getOperand(1);
117 auto Pred = Op->getOperand(2);
118 return CDAG.
getNode(VVPOpcode, LegalResVT, {
LHS,
RHS, Pred, Mask, AVL});
126 const bool IsLoad = (VVPOpc == VEISD::VVP_LOAD);
153 Packing, DataVT.getVectorElementType().getSimpleVT());
156 {Chain, BasePtr, StrideV, Mask, AVL});
158 if (!PassThru || PassThru->
isUndef())
163 {NewLoadV, PassThru, Mask, AVL});
171 assert(VVPOpc == VEISD::VVP_STORE);
172 return CDAG.
getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(),
173 {Chain, Data, BasePtr, StrideV, Mask, AVL});
179 assert((VVPOC == VEISD::VVP_LOAD) || (VVPOC == VEISD::VVP_STORE));
183 "Can only split packed load/store");
187 "Should have been folded in lowering to VVP layer");
196 unsigned ChainResIdx = PackData ? 0 : 1;
208 UpperPartAVL = SplitTM.AVL;
219 CDAG.
getUnpack(SplitDataVT, PackData, Part, SplitTM.AVL);
260 return CDAG.getMergeValues({PackedVals, FusedChains});
278 if (PassThru && PassThru->
isUndef())
281 bool IsScatter = (
bool)StoredValue;
295 {Chain, StoredValue, AddressVec, Mask, AVL});
299 {Chain, AddressVec, Mask, AVL});
306 {NewLoadV, PassThru, Mask, AVL});
329 switch (Op->getOpcode()) {
330 case VEISD::VVP_LOAD:
331 case VEISD::VVP_STORE:
335 EVT IdiomVT = Op.getValueType();
347 auto AVLPos =
getAVLPos(Op->getOpcode());
352 SDValue PackedAVL = AVLPair.first;
353 assert(!AVLPair.second &&
"Expecting non pack-legalized oepration");
365 UpperPartAVL = SplitTM.AVL;
369 for (
unsigned i = 0; i < Op.getNumOperands(); ++i) {
370 if (AVLPos && ((
int)i) == *AVLPos)
372 if (MaskPos && ((
int)i) == *MaskPos)
376 auto PackedOperand = Op.getOperand(i);
377 auto UnpackedOpVT =
splitVectorType(PackedOperand.getSimpleValueType());
379 CDAG.
getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL);
388 CDAG.
getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags());
413 "TODO Shift predication from EVL into Mask");
415 if (
auto *ConstAVL = dyn_cast<ConstantSDNode>(AVL)) {
427 int NumOp = Op->getNumOperands();
428 auto AVLPos =
getAVLPos(Op->getOpcode());
429 std::vector<SDValue> FixedOperands;
430 for (
int i = 0; i < NumOp; ++i) {
431 if (AVLPos && (i == *AVLPos)) {
432 FixedOperands.push_back(AnnotatedLegalAVL);
435 FixedOperands.push_back(Op->getOperand(i));
439 auto Flags = Op->getFlags();
441 CDAG.
getNode(Op->getOpcode(), Op->getVTList(), FixedOperands,
Flags);
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool isUndef() const
Return true if the type of the node type undefined.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVMContext * getContext() const
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
SDValue getSplitPtrOffset(SDValue Ptr, SDValue ByteStride, PackElem Part) const
SDValue getMergeValues(ArrayRef< SDValue > Values) const
} Packing
SDValue getConstantMask(Packing Packing, bool AllTrue) const
SDValue getGatherScatterAddress(SDValue BasePtr, SDValue Scale, SDValue Index, SDValue Mask, SDValue AVL) const
SDValue getLegalReductionOpVVP(unsigned VVPOpcode, EVT ResVT, SDValue StartV, SDValue VectorV, SDValue Mask, SDValue AVL, SDNodeFlags Flags) const
} getNode
SDValue getNode(unsigned OC, SDVTList VTL, ArrayRef< SDValue > OpV, std::optional< SDNodeFlags > Flags=std::nullopt) const
getNode {
SDValue annotateLegalAVL(SDValue AVL) const
SDValue getUnpack(EVT DestVT, SDValue Vec, PackElem Part, SDValue AVL) const
} Legalizing getNode
SDValue getPack(EVT DestVT, SDValue LoVec, SDValue HiVec, SDValue AVL) const
SDValue getConstant(uint64_t Val, EVT VT, bool IsTarget=false, bool IsOpaque=false) const
SDValue getSplitPtrStride(SDValue PackStride) const
VETargetMasks getTargetSplitMask(SDValue RawMask, SDValue RawAVL, PackElem Part) const
SDValue splitMaskArithmetic(SDValue Op, SelectionDAG &DAG) const
SDValue lowerVVP_GATHER_SCATTER(SDValue Op, VECustomDAG &) const
SDValue lowerToVVP(SDValue Op, SelectionDAG &DAG) const
} Custom Inserter
SDValue lowerVVP_LOAD_STORE(SDValue Op, VECustomDAG &) const
SDValue legalizePackedAVL(SDValue Op, VECustomDAG &CDAG) const
SDValue splitPackedLoadStore(SDValue Op, VECustomDAG &CDAG) const
SDValue legalizeInternalVectorOp(SDValue Op, SelectionDAG &DAG) const
SDValue splitVectorOp(SDValue Op, VECustomDAG &CDAG) const
SDValue legalizeInternalLoadStoreOp(SDValue Op, VECustomDAG &CDAG) const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ ADD
Simple integer binary arithmetic operators.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
This is an optimization pass for GlobalISel generic memory operations.
bool isVVPReductionOp(unsigned Opcode)
bool isPackedVectorType(EVT SomeVT)
bool supportsPackedMode(unsigned Opcode, EVT IdiomVT)
std::optional< int > getAVLPos(unsigned Opc)
The VE backend uses a two-staged process to lower and legalize vector instructions:
SDValue getGatherScatterScale(SDValue Op)
SDValue getStoredValue(SDValue Op)
bool isVVPBinaryOp(unsigned VVPOpcode)
std::optional< EVT > getIdiomaticVectorType(SDNode *Op)
} AVL Functions
SDValue getNodeChain(SDValue Op)
Node Properties {.
SDValue getNodeAVL(SDValue Op)
} Node Properties
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isLegalAVL(SDValue AVL)
MVT splitVectorType(MVT VT)
SDValue getNodePassthru(SDValue Op)
bool maySafelyIgnoreMask(SDValue Op)
bool isVVPOrVEC(unsigned Opcode)
MVT getLegalVectorType(Packing P, MVT ElemVT)
SDValue getMemoryPtr(SDValue Op)
std::optional< int > getMaskPos(unsigned Opc)
std::pair< SDValue, bool > getAnnotatedNodeAVL(SDValue Op)
bool hasReductionStartParam(unsigned OPC)
SDValue getGatherScatterIndex(SDValue Op)
Packing getTypePacking(EVT VT)
std::optional< unsigned > getVVPOpcode(unsigned Opcode)
bool isVVPUnaryOp(unsigned VVPOpcode)
SDValue getNodeMask(SDValue Op)
SDValue getLoadStoreStride(SDValue Op, VECustomDAG &CDAG)
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.