35 class VectorLegalizer {
47 LegalizedNodes.
insert(std::make_pair(From, To));
50 LegalizedNodes.insert(std::make_pair(To, To));
133 DAG(dag), TLI(dag.getTargetLoweringInfo()), Changed(
false) {}
136 bool VectorLegalizer::Run() {
138 bool HasVectors =
false;
140 E = std::prev(DAG.allnodes_end());
I != std::next(
E); ++
I) {
145 HasVectors |= J->isVector();
162 DAG.AssignTopologicalOrder();
164 E = std::prev(DAG.allnodes_end());
I != std::next(
E); ++
I)
168 SDValue OldRoot = DAG.getRoot();
169 assert(LegalizedNodes.count(OldRoot) &&
"Root didn't get legalized?");
170 DAG.setRoot(LegalizedNodes[OldRoot]);
172 LegalizedNodes.clear();
175 DAG.RemoveDeadNodes();
191 if (I != LegalizedNodes.
end())
return I->second;
202 bool HasVectorValue =
false;
211 return TranslateLegalizeResults(Op, Result);
213 if (
SDValue Lowered = TLI.LowerOperation(Result, DAG)) {
214 if (Lowered == Result)
215 return TranslateLegalizeResults(Op, Lowered);
222 "There are still live users of the old chain!");
223 return LegalizeOp(Lowered);
225 return TranslateLegalizeResults(Op, Lowered);
229 return LegalizeOp(ExpandLoad(Op));
236 switch (TLI.getTruncStoreAction(ValVT, StVT)) {
239 return TranslateLegalizeResults(Op, Result);
241 SDValue Lowered = TLI.LowerOperation(Result, DAG);
242 Changed = Lowered != Result;
243 return TranslateLegalizeResults(Op, Lowered);
247 return LegalizeOp(ExpandStore(Op));
250 HasVectorValue =
true;
255 HasVectorValue |= J->isVector();
257 return TranslateLegalizeResults(Op, Result);
262 return TranslateLegalizeResults(Op, Result);
338 QueryType = Node->getValueType(0);
341 QueryType = cast<VTSDNode>(Node->getOperand(1))->getVT();
345 QueryType = Node->getOperand(0).getValueType();
348 QueryType = cast<MaskedScatterSDNode>(Node)->getValue().getValueType();
351 QueryType = cast<MaskedStoreSDNode>(Node)->getValue().getValueType();
355 switch (TLI.getOperationAction(Node->getOpcode(), QueryType)) {
358 Result = Promote(Op);
364 if (
SDValue Tmp1 = TLI.LowerOperation(Op, DAG)) {
376 Result = LegalizeOp(Result);
382 AddLegalizedOperand(Op, Result);
393 return PromoteINT_TO_FP(Op);
407 "Can't promote a vector with multiple results!");
418 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
429 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
430 return DAG.getNode(
ISD::FP_ROUND, dl, VT, Op, DAG.getIntPtrConstant(0, dl));
440 "Can't promote a vector with multiple results!");
451 assert(NVT.isSimple() &&
"Promoting to a non-simple vector type!");
459 Operands[j] = DAG.getNode(Opc, dl, NVT, Op.
getOperand(j));
471 SDValue VectorLegalizer::PromoteFP_TO_INT(
SDValue Op,
bool isSigned) {
473 "Can't promote a vector with multiple results!");
480 assert(NewVT.
isSimple() &&
"Promoting to a non-simple vector type!");
485 if (!isSigned && TLI.isOperationLegalOrCustom(
ISD::FP_TO_UINT, NewVT)) {
507 if (SrcVT.getVectorNumElements() > 1 && !SrcEltVT.
isByteSized()) {
523 EVT WideVT = TLI.getPointerTy(DAG.getDataLayout());
526 "Could not handle the sophisticated case when the widest integer is"
529 "Type is not legalized?");
533 unsigned RemainingBytes = SrcVT.getStoreSize();
536 while (RemainingBytes > 0) {
538 unsigned LoadBytes = WideBytes;
540 if (RemainingBytes >= LoadBytes) {
542 DAG.getLoad(WideVT, dl, Chain, BasePTR,
548 while (RemainingBytes < LoadBytes) {
553 DAG.getExtLoad(
ISD::EXTLOAD, dl, WideVT, Chain, BasePTR,
559 RemainingBytes -= LoadBytes;
562 DAG.getConstant(LoadBytes, dl,
571 SDValue SrcEltBitMask = DAG.getConstant((1U << SrcEltBits) - 1, dl, WideVT);
573 unsigned BitOffset = 0;
574 unsigned WideIdx = 0;
577 for (
unsigned Idx = 0; Idx != NumElem; ++Idx) {
580 if (BitOffset < WideBits) {
581 ShAmt = DAG.getConstant(
582 BitOffset, dl, TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
587 BitOffset += SrcEltBits;
588 if (BitOffset >= WideBits) {
590 BitOffset -= WideBits;
592 ShAmt = DAG.getConstant(
593 SrcEltBits - BitOffset, dl,
594 TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
606 Lo = DAG.getAnyExtOrTrunc(Lo, dl, DstEltVT);
609 Lo = DAG.getZExtOrTrunc(Lo, dl, DstEltVT);
613 DAG.getConstant(WideBits - SrcEltBits, dl,
614 TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
617 Lo = DAG.getSExtOrTrunc(Lo, dl, DstEltVT);
627 SDValue Scalarized = TLI.scalarizeVectorLoad(LD, DAG);
633 AddLegalizedOperand(Op.
getValue(0), Value);
634 AddLegalizedOperand(Op.
getValue(1), NewChain);
636 return (Op.
getResNo() ? NewChain : Value);
663 SDValue NewVectorStore = DAG.getTruncStore(
667 ST = cast<StoreSDNode>(NewVectorStore.getNode());
670 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
671 AddLegalizedOperand(Op, TF);
678 return ExpandSEXTINREG(Op);
680 return ExpandANY_EXTEND_VECTOR_INREG(Op);
682 return ExpandSIGN_EXTEND_VECTOR_INREG(Op);
684 return ExpandZERO_EXTEND_VECTOR_INREG(Op);
686 return ExpandBSWAP(Op);
688 return ExpandVSELECT(Op);
690 return ExpandSELECT(Op);
692 return ExpandUINT_TO_FLOAT(Op);
694 return ExpandFNEG(Op);
696 return UnrollVSETCC(Op);
698 return ExpandBITREVERSE(Op);
701 return ExpandCTLZ(Op);
703 return ExpandCTTZ_ZERO_UNDEF(Op);
705 return DAG.UnrollVectorOp(Op.
getNode());
734 return DAG.UnrollVectorOp(Op.
getNode());
742 Mask = DAG.getSelect(DL, BitTy, Mask,
745 DAG.getConstant(0, DL, BitTy));
757 SDValue AllOnes = DAG.getConstant(
773 return DAG.UnrollVectorOp(Op.
getNode());
780 SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
784 return DAG.getNode(
ISD::SRA, DL, VT, Op, ShiftSz);
789 SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(
SDValue Op) {
799 ShuffleMask.
resize(NumSrcElements, -1);
802 int ExtLaneScale = NumSrcElements / NumElements;
803 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
804 for (
int i = 0;
i < NumElements; ++
i)
805 ShuffleMask[
i * ExtLaneScale + EndianOffset] =
i;
809 DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
812 SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(
SDValue Op) {
820 Op = DAG.getAnyExtendVectorInReg(Src, DL, VT);
827 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
829 DAG.getNode(
ISD::SHL, DL, VT, Op, ShiftAmount),
836 SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(
SDValue Op) {
850 ShuffleMask.
reserve(NumSrcElements);
851 for (
int i = 0;
i < NumSrcElements; ++
i)
854 int ExtLaneScale = NumSrcElements / NumElements;
855 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
856 for (
int i = 0;
i < NumElements; ++
i)
857 ShuffleMask[
i * ExtLaneScale + EndianOffset] = NumSrcElements +
i;
860 DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
866 for (
int J = ScalarSizeInBytes - 1; J >= 0; --J)
867 ShuffleMask.
push_back((I * ScalarSizeInBytes) + J);
875 createBSWAPShuffleMask(VT, ShuffleMask);
879 if (!TLI.isShuffleMaskLegal(ShuffleMask, ByteVT))
880 return DAG.UnrollVectorOp(Op.
getNode());
884 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);
893 return DAG.UnrollVectorOp(Op.
getNode());
899 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
901 createBSWAPShuffleMask(VT, BSWAPMask);
904 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
906 (TLI.isOperationLegalOrCustom(
ISD::SHL, ByteVT) &&
907 TLI.isOperationLegalOrCustom(
ISD::SRL, ByteVT) &&
908 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, ByteVT) &&
909 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, ByteVT)))) {
912 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
921 if (!TLI.isOperationLegalOrCustom(
ISD::SHL, VT) ||
922 !TLI.isOperationLegalOrCustom(
ISD::SRL, VT) ||
923 !TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) ||
924 !TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT))
925 return DAG.UnrollVectorOp(Op.
getNode());
954 return DAG.UnrollVectorOp(Op.
getNode());
960 return DAG.UnrollVectorOp(Op.
getNode());
968 SDValue AllOnes = DAG.getConstant(
985 return DAG.UnrollVectorOp(Op.
getNode());
988 assert((BW == 64 || BW == 32) &&
989 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
991 SDValue HalfWord = DAG.getConstant(BW / 2, DL, VT);
996 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
997 SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
1025 return DAG.UnrollVectorOp(Op.
getNode());
1034 TLI.isOperationLegalOrCustom(
ISD::CTLZ, VT)) {
1049 TLI.isOperationLegalOrCustom(
ISD::CTPOP, VT) &&
1050 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1051 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT) &&
1052 TLI.isOperationLegalOrCustomOrPromote(
ISD::XOR, VT)) {
1055 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
1057 for (
unsigned i = 1;
i != NumBitsPerElt;
i *= 2)
1062 Res = DAG.getNOT(DL, Res, VT);
1067 return DAG.UnrollVectorOp(Op.
getNode());
1078 return DAG.UnrollVectorOp(Op.
getNode());
1089 for (
unsigned i = 0;
i < NumElems; ++
i) {
1092 DAG.getConstant(
i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
1095 DAG.getConstant(
i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
1097 TLI.getSetCCResultType(DAG.getDataLayout(),
1098 *DAG.getContext(), TmpEltVT),
1099 LHSElem, RHSElem, CC);
1100 Ops[
i] = DAG.getSelect(dl, EltVT, Ops[
i],
1103 DAG.getConstant(0, dl, EltVT));
1111 return VectorLegalizer(*this).Run();
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
void push_back(const T &Elt)
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
SDValue getValue(unsigned R) const
Flags getFlags() const
Return the raw flags of the source value,.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getNumOperands() const
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned Num) const
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
void reserve(size_type N)
const SDValue & getBasePtr() const
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
unsigned getResNo() const
get the index which selects a specific result in the SDNode
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool isVector() const
isVector - Return true if this is a vector value type.
bool isRound() const
isRound - Return true if the size is a power-of-two number of bytes.
Shift and rotation operations.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
MachinePointerInfo getWithOffset(int64_t O) const
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
bool bitsGE(EVT VT) const
bitsGE - Return true if this has no less bits than VT.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Simple integer binary arithmetic operators.
Function Alias Analysis false
const SDValue & getBasePtr() const
EVT getMemoryVT() const
Return the type of the in-memory value.
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
This class is used to represent ISD::STORE nodes.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
SDNode * getNode() const
get the SDNode which holds the desired result
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
unsigned getScalarSizeInBits() const
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
constexpr bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
MVT - Machine Value Type.
const SDValue & getOperand(unsigned i) const
This is an important class for using LLVM in a threaded context.
Simple binary floating point operators.
bool isVector() const
isVector - Return true if this is a vector value type.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
unsigned getOpcode() const
const SDValue & getValue() const
Bit counting operators with an undefined result for zero inputs.
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
EVT - Extended Value Type.
uint64_t NextPowerOf2(uint64_t A)
NextPowerOf2 - Returns the next power of two (in 64-bits) that is strictly greater than A...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const MachinePointerInfo & getPointerInfo() const
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Iterator for intrusive lists based on ilist_node.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
const SDValue & getChain() const
Byte Swap and Counting operators.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Select(COND, TRUEVAL, FALSEVAL).
ZERO_EXTEND - Used for integer types, zeroing the new bits.
ANY_EXTEND - Used for integer types. The high bits are undefined.
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that when a single input is NaN...
iterator_range< value_op_iterator > op_values() const
Bitwise operators - logical and, logical or, logical xor.
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isByteSized() const
isByteSized - Return true if the bit size is a multiple of 8.
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool isSimple() const
isSimple - Test if the given EVT is simple (as opposed to being extended).
LLVM Value Representation.
FMA - Perform a * b + c with no intermediate rounding step.
bool isTruncatingStore() const
Return true if the op does a truncation before store.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
SetCC operator - This evaluates to a true value iff the condition is true.
MVT getVectorElementType() const
TRUNCATE - Completely drop the high bits.
unsigned getAlignment() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
EVT changeVectorElementTypeToInteger() const
changeVectorElementTypeToInteger - Return a vector with the same number of elements as this vector...
const SDNodeFlags * getFlags() const
This could be defined as a virtual function and implemented more simply and directly, but it is not to avoid creating a vtable for this class.
This file describes how to lower LLVM code to machine code.
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
This class is used to represent ISD::LOAD nodes.