35 class VectorLegalizer {
47 LegalizedNodes.
insert(std::make_pair(From, To));
50 LegalizedNodes.insert(std::make_pair(To, To));
130 DAG(dag), TLI(dag.getTargetLoweringInfo()), Changed(
false) {}
133 bool VectorLegalizer::Run() {
135 bool HasVectors =
false;
137 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I) {
142 HasVectors |= J->isVector();
159 DAG.AssignTopologicalOrder();
161 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I)
165 SDValue OldRoot = DAG.getRoot();
166 assert(LegalizedNodes.count(OldRoot) &&
"Root didn't get legalized?");
167 DAG.setRoot(LegalizedNodes[OldRoot]);
169 LegalizedNodes.clear();
172 DAG.RemoveDeadNodes();
188 if (I != LegalizedNodes.
end())
return I->second;
199 bool HasVectorValue =
false;
208 return TranslateLegalizeResults(Op, Result);
210 if (
SDValue Lowered = TLI.LowerOperation(Result, DAG)) {
211 if (Lowered == Result)
212 return TranslateLegalizeResults(Op, Lowered);
219 "There are still live users of the old chain!");
220 return LegalizeOp(Lowered);
222 return TranslateLegalizeResults(Op, Lowered);
227 return LegalizeOp(ExpandLoad(Op));
234 switch (TLI.getTruncStoreAction(ValVT, StVT.getSimpleVT())) {
237 return TranslateLegalizeResults(Op, Result);
239 SDValue Lowered = TLI.LowerOperation(Result, DAG);
240 Changed = Lowered != Result;
241 return TranslateLegalizeResults(Op, Lowered);
245 return LegalizeOp(ExpandStore(Op));
248 HasVectorValue =
true;
253 HasVectorValue |= J->isVector();
255 return TranslateLegalizeResults(Op, Result);
260 return TranslateLegalizeResults(Op, Result);
329 QueryType = Node->getValueType(0);
332 QueryType = cast<VTSDNode>(Node->getOperand(1))->getVT();
336 QueryType = Node->getOperand(0).getValueType();
339 QueryType = cast<MaskedScatterSDNode>(Node)->getValue().getValueType();
343 switch (TLI.getOperationAction(Node->getOpcode(), QueryType)) {
345 Result = Promote(Op);
351 SDValue Tmp1 = TLI.LowerOperation(Op, DAG);
364 Result = LegalizeOp(Result);
370 AddLegalizedOperand(Op, Result);
381 return PromoteINT_TO_FP(Op);
395 "Can't promote a vector with multiple results!");
406 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint())
417 NVT.isVector() && NVT.getVectorElementType().isFloatingPoint()))
418 return DAG.getNode(
ISD::FP_ROUND, dl, VT, Op, DAG.getIntPtrConstant(0, dl));
428 "Can't promote a vector with multiple results!");
439 assert(NVT.isSimple() &&
"Promoting to a non-simple vector type!");
459 SDValue VectorLegalizer::PromoteFP_TO_INT(
SDValue Op,
bool isSigned) {
461 "Can't promote a vector with multiple results!");
468 assert(NewVT.
isSimple() &&
"Promoting to a non-simple vector type!");
473 if (!isSigned && TLI.isOperationLegalOrCustom(
ISD::FP_TO_UINT, NewVT)) {
506 EVT WideVT = TLI.getPointerTy(DAG.getDataLayout());
509 "Could not handle the sophisticated case when the widest integer is"
511 assert(WideVT.
bitsGE(SrcEltVT) &&
512 "Type is not legalized?");
519 while (RemainingBytes > 0) {
521 unsigned LoadBytes = WideBytes;
523 if (RemainingBytes >= LoadBytes) {
524 ScalarLoad = DAG.getLoad(WideVT, dl, Chain, BasePTR,
532 while (RemainingBytes < LoadBytes) {
536 ScalarLoad = DAG.getExtLoad(
ISD::EXTLOAD, dl, WideVT, Chain, BasePTR,
544 RemainingBytes -= LoadBytes;
547 DAG.getConstant(LoadBytes, dl,
556 SDValue SrcEltBitMask = DAG.getConstant((1U << SrcEltBits) - 1, dl, WideVT);
558 unsigned BitOffset = 0;
559 unsigned WideIdx = 0;
562 for (
unsigned Idx = 0; Idx != NumElem; ++Idx) {
565 if (BitOffset < WideBits) {
566 ShAmt = DAG.getConstant(
567 BitOffset, dl, TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
572 BitOffset += SrcEltBits;
573 if (BitOffset >= WideBits) {
575 BitOffset -= WideBits;
577 ShAmt = DAG.getConstant(
578 SrcEltBits - BitOffset, dl,
579 TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
591 Lo = DAG.getAnyExtOrTrunc(Lo, dl, DstEltVT);
594 Lo = DAG.getZExtOrTrunc(Lo, dl, DstEltVT);
598 DAG.getConstant(WideBits - SrcEltBits, dl,
599 TLI.getShiftAmountTy(WideVT, DAG.getDataLayout()));
602 Lo = DAG.getSExtOrTrunc(Lo, dl, DstEltVT);
610 for (
unsigned Idx=0; Idx<NumElem; Idx++) {
611 SDValue ScalarLoad = DAG.getExtLoad(ExtType, dl,
619 DAG.getConstant(Stride, dl, BasePTR.getValueType()));
630 AddLegalizedOperand(Op.
getValue(0), Value);
631 AddLegalizedOperand(Op.
getValue(1), NewChain);
633 return (Op.
getResNo() ? NewChain : Value);
664 unsigned Stride = ScalarSize/8;
668 for (
unsigned Idx = 0; Idx < NumElem; Idx++) {
671 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
674 SDValue Store = DAG.getTruncStore(Chain, dl, Ex, BasePTR,
685 AddLegalizedOperand(Op, TF);
692 return ExpandSEXTINREG(Op);
694 return ExpandANY_EXTEND_VECTOR_INREG(Op);
696 return ExpandSIGN_EXTEND_VECTOR_INREG(Op);
698 return ExpandZERO_EXTEND_VECTOR_INREG(Op);
700 return ExpandBSWAP(Op);
702 return ExpandVSELECT(Op);
704 return ExpandSELECT(Op);
706 return ExpandUINT_TO_FLOAT(Op);
708 return ExpandFNEG(Op);
710 return UnrollVSETCC(Op);
712 return DAG.UnrollVectorOp(Op.
getNode());
741 return DAG.UnrollVectorOp(Op.
getNode());
749 Mask = DAG.getSelect(DL, BitTy, Mask,
752 DAG.getConstant(0, DL, BitTy));
764 SDValue AllOnes = DAG.getConstant(
780 return DAG.UnrollVectorOp(Op.
getNode());
787 SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
791 return DAG.getNode(
ISD::SRA, DL, VT, Op, ShiftSz);
796 SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(
SDValue Op) {
806 ShuffleMask.
resize(NumSrcElements, -1);
809 int ExtLaneScale = NumSrcElements / NumElements;
810 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
811 for (
int i = 0; i < NumElements; ++i)
812 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
816 DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
819 SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(
SDValue Op) {
827 Op = DAG.getAnyExtendVectorInReg(Src, DL, VT);
834 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
836 DAG.getNode(
ISD::SHL, DL, VT, Op, ShiftAmount),
843 SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(
SDValue Op) {
853 SDValue ScalarZero = DAG.getTargetConstant(0, DL, SrcScalarVT);
860 ShuffleMask.
reserve(NumSrcElements);
861 for (
int i = 0; i < NumSrcElements; ++i)
864 int ExtLaneScale = NumSrcElements / NumElements;
865 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
866 for (
int i = 0; i < NumElements; ++i)
867 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
870 DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
880 for (
int J = ScalarSizeInBytes - 1; J >= 0; --J)
881 ShuffleMask.
push_back((I * ScalarSizeInBytes) + J);
886 if (!TLI.isShuffleMaskLegal(ShuffleMask, ByteVT))
887 return DAG.UnrollVectorOp(Op.
getNode());
891 Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
919 return DAG.UnrollVectorOp(Op.
getNode());
925 return DAG.UnrollVectorOp(Op.
getNode());
933 SDValue AllOnes = DAG.getConstant(
950 return DAG.UnrollVectorOp(Op.
getNode());
954 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
957 SDValue HalfWord = DAG.getConstant(BW/2, DL, VT);
962 uint64_t HWMask = (SVT.
getSizeInBits()==64)?0x00000000FFFFFFFF:0x0000FFFF;
963 SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
990 return DAG.UnrollVectorOp(Op.
getNode());
1001 for (
unsigned i = 0; i < NumElems; ++i) {
1004 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
1007 DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
1009 TLI.getSetCCResultType(DAG.getDataLayout(),
1010 *DAG.getContext(), TmpEltVT),
1011 LHSElem, RHSElem, CC);
1012 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i],
1015 DAG.getConstant(0, dl, EltVT));
1023 return VectorLegalizer(*this).Run();
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
void push_back(const T &Elt)
SDValue getValue(unsigned R) const
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getNumOperands() const
const SDValue & getOperand(unsigned Num) const
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
void reserve(size_type N)
const SDValue & getBasePtr() const
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
unsigned getResNo() const
get the index which selects a specific result in the SDNode
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool isVector() const
isVector - Return true if this is a vector value type.
bool isRound() const
isRound - Return true if the size is a power-of-two number of bytes.
Shift and rotation operations.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
MachinePointerInfo getWithOffset(int64_t O) const
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
bool bitsGE(EVT VT) const
bitsGE - Return true if this has no less bits than VT.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Simple integer binary arithmetic operators.
const SDValue & getBasePtr() const
EVT getMemoryVT() const
Return the type of the in-memory value.
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
SDNode * getNode() const
get the SDNode which holds the desired result
unsigned getScalarSizeInBits() const
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
MVT - Machine Value Type.
const SDValue & getOperand(unsigned i) const
Simple binary floating point operators.
bool isNonTemporal() const
bool isVector() const
isVector - Return true if this is a vector value type.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
unsigned getOpcode() const
const SDValue & getValue() const
Bit counting operators with an undefined result for zero inputs.
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
EVT - Extended Value Type.
uint64_t NextPowerOf2(uint64_t A)
NextPowerOf2 - Returns the next power of two (in 64-bits) that is strictly greater than A...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
const MachinePointerInfo & getPointerInfo() const
TokenFactor - This node takes multiple tokens as input and produces a single token result...
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
const SDValue & getChain() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Byte Swap and Counting operators.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Select(COND, TRUEVAL, FALSEVAL).
ZERO_EXTEND - Used for integer types, zeroing the new bits.
ANY_EXTEND - Used for integer types. The high bits are undefined.
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
iterator_range< value_op_iterator > op_values() const
uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
Bitwise operators - logical and, logical or, logical xor.
pointer data()
Return a pointer to the vector's buffer, even if empty().
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isByteSized() const
isByteSized - Return true if the bit size is a multiple of 8.
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
bool isSimple() const
isSimple - Test if the given EVT is simple (as opposed to being extended).
LLVM Value Representation.
FMA - Perform a * b + c with no intermediate rounding step.
bool isTruncatingStore() const
Return true if the op does a truncation before store.
bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
SetCC operator - This evaluates to a true value iff the condition is true.
MVT getVectorElementType() const
static bool isVolatile(Instruction *Inst)
TRUNCATE - Completely drop the high bits.
unsigned getAlignment() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
EVT changeVectorElementTypeToInteger() const
changeVectorElementTypeToInteger - Return a vector with the same number of elements as this vector...
This file describes how to lower LLVM code to machine code.
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
This class is used to represent ISD::LOAD nodes.