29 #define DEBUG_TYPE "legalize-types" 35 void DAGTypeLegalizer::ScalarizeVectorResult(
SDNode *
N,
unsigned ResNo) {
36 DEBUG(
dbgs() <<
"Scalarize node result " << ResNo <<
": ";
44 dbgs() <<
"ScalarizeVectorResult #" << ResNo <<
": ";
52 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N);
break;
57 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N);
break;
59 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));
break;
62 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N);
break;
63 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N);
break;
65 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N);
break;
66 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N);
break;
71 R = ScalarizeVecRes_VecInregOp(N);
106 R = ScalarizeVecRes_UnaryOp(N);
138 R = ScalarizeVecRes_BinOp(N);
141 R = ScalarizeVecRes_TernaryOp(N);
147 SetScalarizedVector(
SDValue(N, ResNo), R);
157 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(
SDNode *N) {
165 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(
SDNode *N,
167 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
168 return GetScalarizedVector(Op);
171 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(
SDNode *N) {
176 Op = GetScalarizedVector(Op);
182 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(
SDNode *N) {
192 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(
SDNode *N) {
198 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(
SDNode *N) {
211 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(
SDNode *N) {
239 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(
SDNode *N) {
253 Op = GetScalarizedVector(Op);
263 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(
SDNode *N) {
271 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(
SDNode *N) {
280 Op = GetScalarizedVector(Op);
299 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(
SDNode *N) {
309 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(
SDNode *N) {
317 Cond = GetScalarizedVector(Cond);
347 if (ScalarBool != VecBool) {
348 switch (ScalarBool) {
369 auto BoolVT = getSetCCResultType(CondVT);
370 if (BoolVT.bitsLT(CondVT))
378 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(
SDNode *N) {
385 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(
SDNode *N) {
397 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(
SDNode *N) {
402 unsigned Op = !cast<ConstantSDNode>(
Arg)->isNullValue();
403 return GetScalarizedVector(N->
getOperand(Op));
409 "Operand types must be vectors");
412 EVT OpVT = LHS.getValueType();
418 LHS = GetScalarizedVector(LHS);
419 RHS = GetScalarizedVector(RHS);
437 return DAG.
getNode(ExtendCode, DL, NVT, Res);
445 bool DAGTypeLegalizer::ScalarizeVectorOperand(
SDNode *N,
unsigned OpNo) {
446 DEBUG(
dbgs() <<
"Scalarize node operand " << OpNo <<
": ";
451 if (!Res.getNode()) {
455 dbgs() <<
"ScalarizeVectorOperand Op #" << OpNo <<
": ";
462 Res = ScalarizeVecOp_BITCAST(N);
472 Res = ScalarizeVecOp_UnaryOp(N);
475 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
478 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
481 Res = ScalarizeVecOp_VSELECT(N);
484 Res = ScalarizeVecOp_VSETCC(N);
487 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
490 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
496 if (!Res.getNode())
return false;
500 if (Res.getNode() ==
N)
504 "Invalid operand expansion");
506 ReplaceValueWith(
SDValue(N, 0), Res);
512 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(
SDNode *N) {
520 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(
SDNode *N) {
522 "Unexpected vector type!");
532 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(
SDNode *N) {
535 Ops[i] = GetScalarizedVector(N->
getOperand(i));
541 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(
SDNode *N) {
554 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(
SDNode *N) {
568 "Operand types must be vectors");
587 Res = DAG.
getNode(ExtendCode, DL, NVT, Res);
596 assert(OpNo == 1 &&
"Do not know how to scalarize this operand!");
614 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(
SDNode *N,
unsigned OpNo) {
630 void DAGTypeLegalizer::SplitVectorResult(
SDNode *N,
unsigned ResNo) {
643 dbgs() <<
"SplitVectorResult #" << ResNo <<
": ";
652 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi);
break;
654 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi);
break;
655 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi);
break;
661 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi);
break;
667 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
670 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
673 SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
676 SplitVecRes_SETCC(N, Lo, Hi);
679 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
685 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
719 SplitVecRes_UnaryOp(N, Lo, Hi);
725 SplitVecRes_ExtendOp(N, Lo, Hi);
757 SplitVecRes_BinOp(N, Lo, Hi);
760 SplitVecRes_TernaryOp(N, Lo, Hi);
772 GetSplitVector(N->
getOperand(0), LHSLo, LHSHi);
774 GetSplitVector(N->
getOperand(1), RHSLo, RHSHi);
783 void DAGTypeLegalizer::SplitVecRes_TernaryOp(
SDNode *N,
SDValue &Lo,
786 GetSplitVector(N->
getOperand(0), Op0Lo, Op0Hi);
788 GetSplitVector(N->
getOperand(1), Op1Lo, Op1Hi);
790 GetSplitVector(N->
getOperand(2), Op2Lo, Op2Hi);
794 Op0Lo, Op1Lo, Op2Lo);
796 Op0Hi, Op1Hi, Op2Hi);
799 void DAGTypeLegalizer::SplitVecRes_BITCAST(
SDNode *N,
SDValue &Lo,
811 switch (getTypeAction(InVT)) {
825 GetExpandedOp(InOp, Lo, Hi);
836 GetSplitVector(InOp, Lo, Hi);
848 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
856 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(
SDNode *N,
SDValue &Lo,
869 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(
SDNode *N,
SDValue &Lo,
874 if (NumSubvectors == 1) {
890 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(
SDNode *N,
SDValue &Lo,
900 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
906 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(
SDNode *N,
SDValue &Lo,
912 GetSplitVector(Vec, Lo, Hi);
925 unsigned IdxVal = ConstIdx->getZExtValue();
926 if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
953 DAG.
getConstant(IncrementSize, dl, StackPtr.getValueType()));
957 MinAlign(Alignment, IncrementSize));
960 void DAGTypeLegalizer::SplitVecRes_FPOWI(
SDNode *N,
SDValue &Lo,
968 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(
SDNode *N,
SDValue &Lo,
971 GetSplitVector(N->
getOperand(0), LHSLo, LHSHi);
978 GetSplitVector(RHS, RHSLo, RHSHi);
987 void DAGTypeLegalizer::SplitVecRes_InregOp(
SDNode *N,
SDValue &Lo,
990 GetSplitVector(N->
getOperand(0), LHSLo, LHSHi);
994 std::tie(LoVT, HiVT) =
1003 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(
SDNode *N,
SDValue &Lo,
1012 GetSplitVector(N0, InLo, InHi);
1019 EVT OutLoVT, OutHiVT;
1022 assert((2 * OutNumElements) <= InNumElements &&
1023 "Illegal extend vector in reg split");
1033 for (
unsigned i = 0; i != OutNumElements; ++i)
1034 SplitHi[i] = i + OutNumElements;
1037 Lo = DAG.
getNode(Opcode, dl, OutLoVT, InLo);
1038 Hi = DAG.
getNode(Opcode, dl, OutHiVT, InHi);
1041 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(
SDNode *N,
SDValue &Lo,
1047 GetSplitVector(Vec, Lo, Hi);
1050 unsigned IdxVal = CIdx->getZExtValue();
1052 if (IdxVal < LoNumElts)
1099 Lo = DAG.
getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
1103 StackPtr = DAG.
getNode(
ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1105 StackPtr.getValueType()));
1108 Hi = DAG.
getLoad(HiVT, dl, Store, StackPtr,
1109 PtrInfo.getWithOffset(IncrementSize),
1110 MinAlign(Alignment, IncrementSize));
1120 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(
SDNode *N,
SDValue &Lo,
1145 EVT LoMemVT, HiMemVT;
1155 Alignment, MMOFlags, AAInfo);
1164 ReplaceValueWith(
SDValue(LD, 1), Ch);
1182 unsigned SecondHalfAlignment =
1184 Alignment/2 : Alignment;
1189 GetSplitVector(Mask, MaskLo, MaskHi);
1191 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, dl);
1194 EVT LoMemVT, HiMemVT;
1199 GetSplitVector(Src0, Src0Lo, Src0Hi);
1201 std::tie(Src0Lo, Src0Hi) = DAG.
SplitVector(Src0, dl);
1208 Lo = DAG.
getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
1220 Hi = DAG.
getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
1230 ReplaceValueWith(
SDValue(MLD, 1), Ch);
1251 GetSplitVector(Mask, MaskLo, MaskHi);
1253 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, dl);
1256 EVT LoMemVT, HiMemVT;
1262 GetSplitVector(Src0, Src0Lo, Src0Hi);
1264 std::tie(Src0Lo, Src0Hi) = DAG.
SplitVector(Src0, dl);
1268 GetSplitVector(Index, IndexLo, IndexHi);
1270 std::tie(IndexLo, IndexHi) = DAG.
SplitVector(Index, dl);
1277 SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo, Scale};
1281 SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi, Scale};
1292 ReplaceValueWith(
SDValue(MGT, 1), Ch);
1299 "Operand types must be vectors");
1323 void DAGTypeLegalizer::SplitVecRes_UnaryOp(
SDNode *N,
SDValue &Lo,
1347 void DAGTypeLegalizer::SplitVecRes_ExtendOp(
SDNode *N,
SDValue &Lo,
1369 if ((NumElements & 1) == 0 &&
1375 EVT SplitLoVT, SplitHiVT;
1379 DEBUG(
dbgs() <<
"Split vector extend via incremental extend:";
1393 SplitVecRes_UnaryOp(N, Lo, Hi);
1401 GetSplitVector(N->
getOperand(0), Inputs[0], Inputs[1]);
1402 GetSplitVector(N->
getOperand(1), Inputs[2], Inputs[3]);
1417 unsigned InputUsed[2] = { -1U, -1U };
1418 unsigned FirstMaskIdx =
High * NewElts;
1419 bool useBuildVector =
false;
1420 for (
unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1422 int Idx = N->
getMaskElt(FirstMaskIdx + MaskOffset);
1425 unsigned Input = (
unsigned)Idx / NewElts;
1434 Idx -= Input * NewElts;
1439 if (InputUsed[OpNo] == Input) {
1442 }
else if (InputUsed[OpNo] == -1U) {
1444 InputUsed[OpNo] = Input;
1452 useBuildVector =
true;
1460 if (useBuildVector) {
1465 for (
unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1467 int Idx = N->
getMaskElt(FirstMaskIdx + MaskOffset);
1470 unsigned Input = (
unsigned)Idx / NewElts;
1479 Idx -= Input * NewElts;
1489 }
else if (InputUsed[0] == -1U) {
1493 SDValue Op0 = Inputs[InputUsed[0]];
1495 SDValue Op1 = InputUsed[1] == -1U ?
1496 DAG.
getUNDEF(NewVT) : Inputs[InputUsed[1]];
1514 bool DAGTypeLegalizer::SplitVectorOperand(
SDNode *N,
unsigned OpNo) {
1524 if (!Res.getNode()) {
1528 dbgs() <<
"SplitVectorOperand Op #" << OpNo <<
": ";
1535 case ISD::SETCC: Res = SplitVecOp_VSETCC(N);
break;
1541 Res = SplitVecOp_TruncateHelper(N);
1546 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1549 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1552 Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1555 Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1558 Res = SplitVecOp_VSELECT(N, OpNo);
1563 Res = SplitVecOp_TruncateHelper(N);
1565 Res = SplitVecOp_UnaryOp(N);
1570 Res = SplitVecOp_TruncateHelper(N);
1572 Res = SplitVecOp_UnaryOp(N);
1583 Res = SplitVecOp_UnaryOp(N);
1589 Res = SplitVecOp_ExtVecInRegOp(N);
1605 Res = SplitVecOp_VECREDUCE(N, OpNo);
1611 if (!Res.getNode())
return false;
1615 if (Res.getNode() ==
N)
1619 "Invalid operand expansion");
1621 ReplaceValueWith(
SDValue(N, 0), Res);
1625 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(
SDNode *N,
unsigned OpNo) {
1628 assert(OpNo == 0 &&
"Illegal operand must be mask");
1639 assert(Lo.getValueType() == Hi.getValueType() &&
1640 "Lo and Hi have differing types");
1644 assert(LoOpVT == HiOpVT &&
"Asymmetric vector split?");
1646 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
1647 std::tie(LoOp0, HiOp0) = DAG.
SplitVector(Src0, DL);
1648 std::tie(LoOp1, HiOp1) = DAG.
SplitVector(Src1, DL);
1649 std::tie(LoMask, HiMask) = DAG.
SplitVector(Mask, DL);
1659 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(
SDNode *N,
unsigned OpNo) {
1666 assert(VecVT.
isVector() &&
"Can only split reduce vector operand");
1667 GetSplitVector(VecOp, Lo, Hi);
1672 unsigned CombineOpc = 0;
1724 Lo = BitConvertToInteger(Lo);
1725 Hi = BitConvertToInteger(Hi);
1731 JoinIntegers(Lo, Hi));
1734 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(
SDNode *N) {
1743 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1745 if (IdxVal < LoElts) {
1747 "Extracted subvector crosses vector split!");
1756 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(
SDNode *N) {
1761 if (isa<ConstantSDNode>(Idx)) {
1762 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1766 GetSplitVector(Vec, Lo, Hi);
1768 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1770 if (IdxVal < LoElts)
1805 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(
SDNode *N) {
1810 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
1832 GetSplitVector(Mask, MaskLo, MaskHi);
1834 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, dl);
1837 EVT LoMemVT, HiMemVT;
1842 GetSplitVector(Src0, Src0Lo, Src0Hi);
1844 std::tie(Src0Lo, Src0Hi) = DAG.
SplitVector(Src0, dl);
1848 GetSplitVector(Index, IndexLo, IndexHi);
1850 std::tie(IndexLo, IndexHi) = DAG.
SplitVector(Index, dl);
1857 SDValue OpsLo[] = {Ch, Src0Lo, MaskLo, Ptr, IndexLo, Scale};
1867 SDValue OpsHi[] = {Ch, Src0Hi, MaskHi, Ptr, IndexHi, Scale};
1878 ReplaceValueWith(
SDValue(MGT, 1), Ch);
1882 ReplaceValueWith(
SDValue(MGT, 0), Res);
1896 EVT LoMemVT, HiMemVT;
1902 GetSplitVector(Data, DataLo, DataHi);
1904 std::tie(DataLo, DataHi) = DAG.
SplitVector(Data, DL);
1909 GetSplitVector(Mask, MaskLo, MaskHi);
1911 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, DL);
1915 unsigned SecondHalfAlignment =
1917 Alignment/2 : Alignment;
1925 Lo = DAG.
getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
1938 Hi = DAG.
getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
1959 EVT LoMemVT, HiMemVT;
1965 GetSplitVector(Data, DataLo, DataHi);
1967 std::tie(DataLo, DataHi) = DAG.
SplitVector(Data, DL);
1972 GetSplitVector(Mask, MaskLo, MaskHi);
1974 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, DL);
1978 GetSplitVector(Index, IndexLo, IndexHi);
1980 std::tie(IndexLo, IndexHi) = DAG.
SplitVector(Index, DL);
1988 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2000 SDValue OpsHi[] = {
Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2007 assert(OpNo == 1 &&
"Can only split the stored value");
2020 EVT LoMemVT, HiMemVT;
2024 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2027 unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
2031 Alignment, MMOFlags, AAInfo);
2042 HiMemVT, Alignment, MMOFlags, AAInfo);
2046 Alignment, MMOFlags, AAInfo);
2051 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(
SDNode *N) {
2062 for (
unsigned i = 0, e =
Op.getValueType().getVectorNumElements();
2073 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(
SDNode *N) {
2099 assert(!(NumElements & 1) &&
"Splitting vector, but not in half!");
2107 if (InElementSize <= OutElementSize * 2)
2108 return SplitVecOp_UnaryOp(N);
2113 GetSplitVector(InVec, InLoVec, InHiVec);
2115 EVT HalfElementVT = IsFloat ?
2140 "Operand types must be vectors");
2142 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2146 unsigned PartElements = Lo0.getValueType().getVectorNumElements();
2185 void DAGTypeLegalizer::WidenVectorResult(
SDNode *N,
unsigned ResNo) {
2186 DEBUG(
dbgs() <<
"Widen node result " << ResNo <<
": ";
2198 dbgs() <<
"WidenVectorResult #" << ResNo <<
": ";
2205 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N);
break;
2211 case ISD::LOAD: Res = WidenVecRes_LOAD(N);
break;
2215 case ISD::SELECT: Res = WidenVecRes_SELECT(N);
break;
2217 case ISD::SETCC: Res = WidenVecRes_SETCC(N);
break;
2218 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N);
break;
2220 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2223 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2226 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2245 Res = WidenVecRes_Binary(N);
2258 Res = WidenVecRes_BinaryCanTrap(N);
2262 Res = WidenVecRes_FCOPYSIGN(N);
2266 Res = WidenVecRes_POWI(N);
2272 Res = WidenVecRes_Shift(N);
2278 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2291 Res = WidenVecRes_Convert(N);
2315 Res = WidenVecRes_Unary(N);
2318 Res = WidenVecRes_Ternary(N);
2324 SetWidenedVector(
SDValue(N, ResNo), Res);
2346 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(
SDNode *N) {
2356 NumElts = NumElts / 2;
2378 unsigned ConcatEnd = 0;
2386 while (CurNumElts != 0) {
2387 while (CurNumElts >= NumElts) {
2394 ConcatOps[ConcatEnd++] = DAG.
getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
2396 CurNumElts -= NumElts;
2399 NumElts = NumElts / 2;
2404 for (
unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
2411 ConcatOps[ConcatEnd++] = DAG.
getNode(Opcode, dl, WidenEltVT,
2419 if (ConcatEnd == 1) {
2420 VT = ConcatOps[0].getValueType();
2422 return ConcatOps[0];
2429 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2430 Idx = ConcatEnd - 1;
2431 VT = ConcatOps[Idx--].getValueType();
2432 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2445 unsigned NumToInsert = ConcatEnd - Idx - 1;
2446 for (
unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
2451 ConcatOps[Idx+1] = VecOp;
2452 ConcatEnd = Idx + 2;
2458 unsigned RealVals = ConcatEnd - Idx - 1;
2459 unsigned SubConcatEnd = 0;
2460 unsigned SubConcatIdx = Idx + 1;
2461 while (SubConcatEnd < RealVals)
2462 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2463 while (SubConcatEnd < OpsToConcat)
2464 SubConcatOps[SubConcatEnd++] = undefVec;
2466 NextVT, SubConcatOps);
2467 ConcatEnd = SubConcatIdx + 1;
2472 if (ConcatEnd == 1) {
2473 VT = ConcatOps[0].getValueType();
2475 return ConcatOps[0];
2480 if (NumOps != ConcatEnd ) {
2482 for (
unsigned j = ConcatEnd; j < NumOps; ++j)
2483 ConcatOps[j] = UndefVal;
2506 InVTNumElts = InVT.getVectorNumElements();
2507 if (InVTNumElts == WidenNumElts) {
2509 return DAG.
getNode(Opcode, DL, WidenVT, InOp);
2529 if (WidenNumElts % InVTNumElts == 0) {
2531 unsigned NumConcat = WidenNumElts/InVTNumElts;
2535 for (
unsigned i = 1; i != NumConcat; ++i)
2539 return DAG.
getNode(Opcode, DL, WidenVT, InVec);
2543 if (InVTNumElts % WidenNumElts == 0) {
2549 return DAG.
getNode(Opcode, DL, WidenVT, InVal);
2557 unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
2559 for (i=0; i < MinElts; ++i) {
2564 Ops[i] = DAG.
getNode(Opcode, DL, EltVT, Val);
2570 for (; i < WidenNumElts; ++i)
2576 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(
SDNode *N) {
2590 InOp = GetWidenedVector(InOp);
2606 for (
unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
2625 while (Ops.
size() != WidenNumElts)
2631 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(
SDNode *N) {
2635 return WidenVecRes_BinaryCanTrap(N);
2654 EVT ShVT = ShOp.getValueType();
2656 ShOp = GetWidenedVector(ShOp);
2657 ShVT = ShOp.getValueType();
2662 if (ShVT != ShWidenVT)
2663 ShOp = ModifyToType(ShOp, ShWidenVT);
2679 .getVectorElementType(),
2686 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(
SDNode *N,
unsigned ResNo) {
2687 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
2688 return GetWidenedVector(WidenVec);
2698 switch (getTypeAction(InVT)) {
2710 InOp = GetPromotedInteger(InOp);
2712 if (WidenVT.
bitsEq(InVT))
2725 InOp = GetWidenedVector(InOp);
2727 if (WidenVT.
bitsEq(InVT))
2736 if (WidenSize % InSize == 0 && InVT !=
MVT::x86mmx) {
2741 unsigned NewNumElts = WidenSize / InSize;
2759 for (
unsigned i = 1; i < NewNumElts; ++i)
2771 return CreateStackStoreLoad(InOp, WidenVT);
2774 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(
SDNode *N) {
2788 assert(WidenNumElts >= NumElts &&
"Shrinking vector instead of widening!");
2789 NewOps.append(WidenNumElts - NumElts, DAG.
getUNDEF(EltVT));
2794 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(
SDNode *N) {
2802 bool InputWidened =
false;
2810 for (
unsigned i=0; i < NumOperands; ++i)
2812 for (
unsigned i = NumOperands; i != NumConcat; ++i)
2817 InputWidened =
true;
2821 for (i=1; i < NumOperands; ++i)
2825 if (i == NumOperands)
2830 if (NumOperands == 2) {
2833 for (
unsigned i = 0; i < NumInElts; ++i) {
2835 MaskOps[i + NumInElts] = i + WidenNumElts;
2849 for (
unsigned i=0; i < NumOperands; ++i) {
2852 InOp = GetWidenedVector(InOp);
2853 for (
unsigned j=0; j < NumInElts; ++j)
2859 for (; Idx < WidenNumElts; ++Idx)
2860 Ops[Idx] = UndefVal;
2864 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(
SDNode *N) {
2873 InOp = GetWidenedVector(InOp);
2878 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2879 if (IdxVal == 0 && InVT == WidenVT)
2884 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
2893 for (i=0; i < NumElts; ++i)
2900 for (; i < WidenNumElts; ++i)
2905 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(
SDNode *N) {
2919 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
2921 Result = GenWidenVectorLoads(LdChain, LD);
2927 if (LdChain.
size() == 1)
2928 NewChain = LdChain[0];
2934 ReplaceValueWith(
SDValue(N, 1), NewChain);
2952 Mask = ModifyToType(Mask, WideMaskVT,
true);
2978 Mask = ModifyToType(Mask, WideMaskVT,
true);
2985 Index = ModifyToType(Index, WideIndexVT);
2997 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(
SDNode *N) {
3061 if (MaskScalarBits < ToMaskScalBits) {
3065 }
else if (MaskScalarBits > ToMaskScalBits) {
3073 "Mask should have the right element size by now.");
3091 "A mask of ToMaskVT should have been produced by now.");
3097 EVT DAGTypeLegalizer::getSETCCWidenedResultTy(
SDValue SetCC) {
3132 EVT FinalVT = VSelVT;
3144 EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
3162 VSelOp1 = GetWidenedVector(VSelOp1);
3163 VSelOp2 = GetWidenedVector(VSelOp2);
3167 EVT ToMaskVT = VSelVT;
3173 EVT MaskVT = getSETCCWidenedResultTy(Cond);
3174 Mask = convertMask(Cond, MaskVT, ToMaskVT);
3181 EVT VT0 = getSETCCWidenedResultTy(SETCC0);
3182 EVT VT1 = getSETCCWidenedResultTy(SETCC1);
3190 if (ScalarBits0 != ScalarBits1) {
3191 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
3192 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
3204 SETCC0 = convertMask(SETCC0, VT0, MaskVT);
3205 SETCC1 = convertMask(SETCC1, VT1, MaskVT);
3209 Mask = convertMask(Cond, MaskVT, ToMaskVT);
3223 if (
SDValue Res = WidenVSELECTAndMask(N))
3228 CondEltVT, WidenNumElts);
3230 Cond1 = GetWidenedVector(Cond1);
3238 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
3239 SDValue Res = ModifyToType(SplitSelect, WidenVT);
3244 Cond1 = ModifyToType(Cond1, CondWidenVT);
3251 WidenVT, Cond1, InOp1, InOp2);
3254 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(
SDNode *N) {
3280 for (
unsigned i = 0; i != NumElts; ++i) {
3282 if (Idx < (
int)NumElts)
3283 NewMask.push_back(Idx);
3285 NewMask.push_back(Idx - NumElts + WidenNumElts);
3287 for (
unsigned i = NumElts; i != WidenNumElts; ++i)
3288 NewMask.push_back(-1);
3295 "Operands must be vectors");
3309 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
3310 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
3314 InOp1 = GetWidenedVector(InOp1);
3321 "Input not widened to expected type!");
3331 bool DAGTypeLegalizer::WidenVectorOperand(
SDNode *N,
unsigned OpNo) {
3332 DEBUG(
dbgs() <<
"Widen node operand " << OpNo <<
": ";
3344 dbgs() <<
"WidenVectorOperand op #" << OpNo <<
": ";
3354 case ISD::STORE: Res = WidenVecOp_STORE(N);
break;
3355 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo);
break;
3356 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo);
break;
3357 case ISD::SETCC: Res = WidenVecOp_SETCC(N);
break;
3363 Res = WidenVecOp_EXTEND(N);
3372 Res = WidenVecOp_Convert(N);
3377 if (!Res.getNode())
return false;
3381 if (Res.getNode() ==
N)
3386 "Invalid operand expansion");
3388 ReplaceValueWith(
SDValue(N, 0), Res);
3399 "Unexpected type action");
3400 InOp = GetWidenedVector(InOp);
3403 "Input wasn't widened!");
3415 FixedEltVT == InEltVT) {
3417 "Not enough elements in the fixed type for the operand!");
3419 "We can't have the same type as we started with!");
3436 return WidenVecOp_Convert(N);
3469 "Unexpected type action");
3470 InOp = GetWidenedVector(InOp);
3487 for (
unsigned i=0; i < NumElts; ++i)
3504 unsigned InWidenSize = InWidenVT.getSizeInBits();
3508 unsigned NewNumElts = InWidenSize /
Size;
3518 return CreateStackStoreLoad(InOp, VT);
3521 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(
SDNode *N) {
3536 for (
unsigned i=0; i < NumOperands; ++i) {
3540 "Unexpected type action");
3541 InOp = GetWidenedVector(InOp);
3542 for (
unsigned j=0; j < NumInElts; ++j)
3550 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(
SDNode *N) {
3556 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(
SDNode *N) {
3572 GenWidenVectorTruncStores(StChain, ST);
3574 GenWidenVectorStores(StChain, ST);
3576 if (StChain.
size() == 1)
3582 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(
SDNode *N,
unsigned OpNo) {
3583 assert(OpNo == 3 &&
"Can widen only data operand of mstore");
3586 EVT MaskVT = Mask.getValueType();
3589 SDValue WideVal = GetWidenedVector(StVal);
3597 Mask = ModifyToType(Mask, WideMaskVT,
true);
3599 assert(Mask.getValueType().getVectorNumElements() ==
3601 "Mask and data vectors should have the same number of elements");
3607 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(
SDNode *N,
unsigned OpNo) {
3608 assert(OpNo == 1 &&
"Can widen only data operand of mscatter");
3616 SDValue WideVal = GetWidenedVector(DataOp);
3624 Mask = ModifyToType(Mask, WideMaskVT,
true);
3631 Index = ModifyToType(Index, WideIndexVT);
3670 return PromoteTargetBoolean(CC, VT);
3686 unsigned Width,
EVT WidenVT,
3687 unsigned Align = 0,
unsigned WidenEx = 0) {
3691 unsigned AlignInBits =
Align*8;
3694 EVT RetVT = WidenEltVT;
3695 if (Width == WidenEltWidth)
3709 (WidenWidth % MemVTWidth) == 0 &&
3711 (MemVTWidth <= Width ||
3712 (
Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3725 (WidenWidth % MemVTWidth) == 0 &&
3727 (MemVTWidth <= Width ||
3728 (
Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3743 unsigned Start,
unsigned End) {
3745 SDLoc dl(LdOps[Start]);
3746 EVT LdTy = LdOps[Start].getValueType();
3754 for (
unsigned i = Start + 1; i !=
End; ++i) {
3755 EVT NewLdTy = LdOps[i].getValueType();
3756 if (NewLdTy != LdTy) {
3792 int WidthDiff = WidenWidth - LdWidth;
3796 EVT NewVT =
FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3799 Align, MMOFlags, AAInfo);
3803 if (LdWidth <= NewVTWidth) {
3805 unsigned NumElts = WidenWidth / NewVTWidth;
3810 if (NewVT == WidenVT)
3813 assert(WidenWidth % NewVTWidth == 0);
3814 unsigned NumConcat = WidenWidth / NewVTWidth;
3817 ConcatOps[0] = LdOp;
3818 for (
unsigned i = 1; i != NumConcat; ++i)
3819 ConcatOps[i] = UndefVal;
3827 LdWidth -= NewVTWidth;
3830 while (LdWidth > 0) {
3831 unsigned Increment = NewVTWidth / 8;
3832 Offset += Increment;
3836 if (LdWidth < NewVTWidth) {
3838 NewVT =
FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3840 L = DAG.
getLoad(NewVT, dl, Chain, BasePtr,
3842 MinAlign(Align, Increment), MMOFlags, AAInfo);
3851 while (size < LdOp->getValueSizeInBits(0)) {
3858 L = DAG.
getLoad(NewVT, dl, Chain, BasePtr,
3860 MinAlign(Align, Increment), MMOFlags, AAInfo);
3867 LdWidth -= NewVTWidth;
3872 if (!LdOps[0].getValueType().
isVector())
3882 EVT LdTy = LdOps[i].getValueType();
3885 for (--i; i >= 0; --i) {
3886 LdTy = LdOps[i].getValueType();
3892 ConcatOps[--Idx] = LdOps[i];
3893 for (--i; i >= 0; --i) {
3894 EVT NewLdTy = LdOps[i].getValueType();
3895 if (NewLdTy != LdTy) {
3902 ConcatOps[--Idx] = LdOps[i];
3915 for (; i != End-Idx; ++i)
3916 WidenOps[i] = ConcatOps[Idx+i];
3917 for (; i != NumOps; ++i)
3918 WidenOps[i] = UndefVal;
3951 LdEltVT,
Align, MMOFlags, AAInfo);
3953 unsigned i = 0,
Offset = Increment;
3954 for (i=1; i < NumElts; ++i,
Offset += Increment) {
3956 Ops[i] = DAG.
getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
3958 Align, MMOFlags, AAInfo);
3964 for (; i != WidenNumElts; ++i)
3993 while (StWidth != 0) {
3997 unsigned Increment = NewVTWidth / 8;
4006 MinAlign(Align, Offset), MMOFlags, AAInfo));
4007 StWidth -= NewVTWidth;
4008 Offset += Increment;
4012 }
while (StWidth != 0 && StWidth >= NewVTWidth);
4015 unsigned NumElts = ValWidth / NewVTWidth;
4019 Idx = Idx * ValEltWidth / NewVTWidth;
4027 MinAlign(Align, Offset), MMOFlags, AAInfo));
4028 StWidth -= NewVTWidth;
4029 Offset += Increment;
4031 }
while (StWidth != 0 && StWidth >= NewVTWidth);
4033 Idx = Idx * NewVTWidth / ValEltWidth;
4071 unsigned Offset = Increment;
4072 for (
unsigned i=1; i < NumElts; ++i, Offset += Increment) {
4079 StEltVT,
MinAlign(Align, Offset), MMOFlags, AAInfo));
4087 bool FillWithZeroes) {
4092 "input and widen element type must match");
4101 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
4102 unsigned NumConcat = WidenNumElts / InNumElts;
4107 for (
unsigned i = 1; i != NumConcat; ++i)
4113 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
4121 unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
4123 for (Idx = 0; Idx < MinNumElts; ++Idx)
4130 for ( ; Idx < WidenNumElts; ++Idx)
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
void push_back(const T &Elt)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Compute iterated dominance frontiers using a linear time algorithm.
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
SDValue getSignExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will sign extend the low lanes of the operand into the specified vector typ...
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
const SDValue & getBasePtr() const
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
static ISD::NodeType getExtendForContent(BooleanContent Content)
const SDValue & getChain() const
unsigned getAlignment() const
unsigned getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
bool isInteger() const
Return true if this is an integer or a vector integer type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
const SDNodeFlags getFlags() const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
static SDValue BuildVectorFromScalar(SelectionDAG &DAG, EVT VecTy, SmallVectorImpl< SDValue > &LdOps, unsigned Start, unsigned End)
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
const SDValue & getValue() const
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
bool isTruncatingStore() const
Return true if the op does a truncation before store.
SDValue getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will any-extend the low lanes of the operand into the specified vector type...
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
A description of a memory reference used in the backend.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Shift and rotation operations.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
op_iterator op_end() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
const DataLayout & getDataLayout() const
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getScalarSizeInBits() const
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
MachineFunction & getMachineFunction() const
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
const SDValue & getScale() const
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Simple integer binary arithmetic operators.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
UNDEF - An undefined node.
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getBasePtr() const
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
These reductions are non-strict, and have a single vector operand.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
bool isExpandingLoad() const
static EVT FindMemType(SelectionDAG &DAG, const TargetLowering &TLI, unsigned Width, EVT WidenVT, unsigned Align=0, unsigned WidenEx=0)
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
Simple binary floating point operators.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
iterator_range< value_op_iterator > op_values() const
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Idx) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
static const unsigned End
unsigned getOriginalAlignment() const
Returns alignment and volatility of the memory access.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
Bit counting operators with an undefined result for zero inputs.
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
const SDValue & getIndex() const
const SDValue & getBasePtr() const
const SDValue & getSrc0() const
This class contains a discriminated union of information about pointers in memory operands...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
const SDValue & getMask() const
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
The memory access writes data.
SDValue getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will zero extend the low lanes of the operand into the specified vector typ...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
void dump() const
Dump this node, for debugging.
const TargetLowering & getTargetLoweringInfo() const
Returns platform specific canonical encoding of a floating point number.
virtual bool canOpTrap(unsigned Op, EVT VT) const
Returns true if the operation can trap for the value type.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
EVT getVectorElementType() const
Given a vector type, return the type of each element.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part...
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
bool isVector(MCInstrInfo const &MCII, MCInst const &MCI)
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Byte Swap and Counting operators.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const
Increments memory address Addr according to the type of the value DataVT that should be stored...
Represents one node in the SelectionDAG.
static bool isSETCCorConvertedSETCC(SDValue N)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
MachinePointerInfo getWithOffset(int64_t O) const
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Select(COND, TRUEVAL, FALSEVAL).
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ZERO_EXTEND - Used for integer types, zeroing the new bits.
ANY_EXTEND - Used for integer types. The high bits are undefined.
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
int getMaskElt(unsigned Idx) const
FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that when a single input is NaN...
Flags
Flags values. These may be or'd together.
amdgpu Simplify well known AMD library false Value Value * Arg
The memory access reads data.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
bool isVector() const
Return true if this is a vector value type.
Bitwise operators - logical and, logical or, logical xor.
pointer data()
Return a pointer to the vector's buffer, even if empty().
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
Flags getFlags() const
Return the raw flags of the source value,.
unsigned getOpcode() const
SDValue getValue(unsigned R) const
This class is used to represent an MSCATTER node.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
const MachinePointerInfo & getPointerInfo() const
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This class is used to represent an MLOAD node.
FMA - Perform a * b + c with no intermediate rounding step.
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isUndef() const
Return true if the type of the node type undefined.
ISD::LoadExtType getExtensionType() const
SetCC operator - This evaluates to a true value iff the condition is true.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
static bool isLogicalMaskOp(unsigned Opcode)
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
BooleanContent
Enum that describes how the target represents true/false values.
const SDValue & getOperand(unsigned i) const
TRUNCATE - Completely drop the high bits.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object...
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
LLVMContext * getContext() const
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
const SDValue & getMask() const
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
This class is used to represent ISD::LOAD nodes.