29 #define DEBUG_TYPE "legalize-types"
35 void DAGTypeLegalizer::ScalarizeVectorResult(
SDNode *
N,
unsigned ResNo) {
36 DEBUG(
dbgs() <<
"Scalarize node result " << ResNo <<
": ";
44 dbgs() <<
"ScalarizeVectorResult #" << ResNo <<
": ";
52 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N);
break;
57 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N);
break;
59 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));
break;
62 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N);
break;
63 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N);
break;
65 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N);
break;
66 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N);
break;
100 R = ScalarizeVecRes_UnaryOp(N);
132 R = ScalarizeVecRes_BinOp(N);
135 R = ScalarizeVecRes_TernaryOp(N);
141 SetScalarizedVector(
SDValue(N, ResNo), R);
151 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(
SDNode *N) {
159 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(
SDNode *N,
161 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
162 return GetScalarizedVector(Op);
165 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(
SDNode *N) {
171 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(
SDNode *N) {
181 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(
SDNode *N) {
187 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(
SDNode *N) {
200 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(
SDNode *N) {
228 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(
SDNode *N) {
242 Op = GetScalarizedVector(Op);
252 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(
SDNode *N) {
260 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(
SDNode *N) {
270 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(
SDNode *N) {
293 if (ScalarBool != VecBool) {
295 switch (ScalarBool) {
316 LHS.getValueType(), Cond, LHS,
320 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(
SDNode *N) {
327 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(
SDNode *N) {
338 "Scalar/Vector type mismatch");
354 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(
SDNode *N) {
359 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
360 return GetScalarizedVector(N->
getOperand(Op));
363 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(
SDNode *N) {
366 "Operand types must be vectors");
369 EVT OpVT = LHS.getValueType();
375 LHS = GetScalarizedVector(LHS);
376 RHS = GetScalarizedVector(RHS);
394 return DAG.
getNode(ExtendCode, DL, NVT, Res);
402 bool DAGTypeLegalizer::ScalarizeVectorOperand(
SDNode *N,
unsigned OpNo) {
403 DEBUG(
dbgs() <<
"Scalarize node operand " << OpNo <<
": ";
408 if (!Res.getNode()) {
412 dbgs() <<
"ScalarizeVectorOperand Op #" << OpNo <<
": ";
418 Res = ScalarizeVecOp_BITCAST(N);
428 Res = ScalarizeVecOp_UnaryOp(N);
431 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
434 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
437 Res = ScalarizeVecOp_VSELECT(N);
440 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
443 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
449 if (!Res.getNode())
return false;
453 if (Res.getNode() ==
N)
457 "Invalid operand expansion");
459 ReplaceValueWith(
SDValue(N, 0), Res);
465 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(
SDNode *N) {
473 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(
SDNode *N) {
475 "Unexpected vector type!");
485 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(
SDNode *N) {
494 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(
SDNode *N) {
506 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(
SDNode *N) {
518 assert(OpNo == 1 &&
"Do not know how to scalarize this operand!");
536 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(
SDNode *N,
unsigned OpNo) {
552 void DAGTypeLegalizer::SplitVectorResult(
SDNode *N,
unsigned ResNo) {
565 dbgs() <<
"SplitVectorResult #" << ResNo <<
": ";
574 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi);
break;
576 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi);
break;
577 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi);
break;
583 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi);
break;
589 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
592 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
595 SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
598 SplitVecRes_SETCC(N, Lo, Hi);
601 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
607 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
640 SplitVecRes_UnaryOp(N, Lo, Hi);
646 SplitVecRes_ExtendOp(N, Lo, Hi);
678 SplitVecRes_BinOp(N, Lo, Hi);
681 SplitVecRes_TernaryOp(N, Lo, Hi);
690 void DAGTypeLegalizer::SplitVecRes_BinOp(
SDNode *N,
SDValue &Lo,
693 GetSplitVector(N->
getOperand(0), LHSLo, LHSHi);
695 GetSplitVector(N->
getOperand(1), RHSLo, RHSHi);
704 void DAGTypeLegalizer::SplitVecRes_TernaryOp(
SDNode *N,
SDValue &Lo,
707 GetSplitVector(N->
getOperand(0), Op0Lo, Op0Hi);
709 GetSplitVector(N->
getOperand(1), Op1Lo, Op1Hi);
711 GetSplitVector(N->
getOperand(2), Op2Lo, Op2Hi);
715 Op0Lo, Op1Lo, Op2Lo);
717 Op0Hi, Op1Hi, Op2Hi);
720 void DAGTypeLegalizer::SplitVecRes_BITCAST(
SDNode *N,
SDValue &Lo,
732 switch (getTypeAction(InVT)) {
746 GetExpandedOp(InOp, Lo, Hi);
757 GetSplitVector(InOp, Lo, Hi);
769 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
777 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(
SDNode *N,
SDValue &Lo,
790 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(
SDNode *N,
SDValue &Lo,
795 if (NumSubvectors == 1) {
811 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(
SDNode *N,
SDValue &Lo,
821 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
827 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(
SDNode *N,
SDValue &Lo,
833 GetSplitVector(Vec, Lo, Hi);
846 unsigned IdxVal = ConstIdx->getZExtValue();
847 if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
874 DAG.
getConstant(IncrementSize, dl, StackPtr.getValueType()));
878 MinAlign(Alignment, IncrementSize));
881 void DAGTypeLegalizer::SplitVecRes_FPOWI(
SDNode *N,
SDValue &Lo,
889 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(
SDNode *N,
SDValue &Lo,
892 GetSplitVector(N->
getOperand(0), LHSLo, LHSHi);
899 GetSplitVector(RHS, RHSLo, RHSHi);
908 void DAGTypeLegalizer::SplitVecRes_InregOp(
SDNode *N,
SDValue &Lo,
911 GetSplitVector(N->
getOperand(0), LHSLo, LHSHi);
915 std::tie(LoVT, HiVT) =
924 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(
SDNode *N,
SDValue &Lo,
931 GetSplitVector(N0, InLo, InHi);
935 EVT OutLoVT, OutHiVT;
938 assert((2 * OutNumElements) <= InNumElements &&
939 "Illegal extend vector in reg split");
949 for (
unsigned i = 0;
i != OutNumElements; ++
i)
950 SplitHi[
i] =
i + OutNumElements;
953 Lo = DAG.
getNode(Opcode, dl, OutLoVT, InLo);
954 Hi = DAG.
getNode(Opcode, dl, OutHiVT, InHi);
957 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(
SDNode *N,
SDValue &Lo,
963 GetSplitVector(Vec, Lo, Hi);
966 unsigned IdxVal = CIdx->getZExtValue();
968 if (IdxVal < LoNumElts)
1004 StackPtr = DAG.
getNode(
ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1006 StackPtr.getValueType()));
1010 MinAlign(Alignment, IncrementSize));
1013 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(
SDNode *N,
SDValue &Lo,
1038 EVT LoMemVT, HiMemVT;
1049 Alignment, MMOFlags, AAInfo);
1058 ReplaceValueWith(
SDValue(LD, 1), Ch);
1076 unsigned SecondHalfAlignment =
1078 Alignment/2 : Alignment;
1083 GetSplitVector(Mask, MaskLo, MaskHi);
1085 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, dl);
1088 EVT LoMemVT, HiMemVT;
1093 GetSplitVector(Src0, Src0Lo, Src0Hi);
1095 std::tie(Src0Lo, Src0Hi) = DAG.
SplitVector(Src0, dl);
1102 Lo = DAG.
getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
1113 Hi = DAG.
getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
1124 ReplaceValueWith(
SDValue(MLD, 1), Ch);
1144 GetSplitVector(Mask, MaskLo, MaskHi);
1146 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, dl);
1149 EVT LoMemVT, HiMemVT;
1155 GetSplitVector(Src0, Src0Lo, Src0Hi);
1157 std::tie(Src0Lo, Src0Hi) = DAG.
SplitVector(Src0, dl);
1161 GetSplitVector(Index, IndexLo, IndexHi);
1163 std::tie(IndexLo, IndexHi) = DAG.
SplitVector(Index, dl);
1170 SDValue OpsLo[] = {Ch, Src0Lo, MaskLo,
Ptr, IndexLo};
1174 SDValue OpsHi[] = {Ch, Src0Hi, MaskHi,
Ptr, IndexHi};
1185 ReplaceValueWith(
SDValue(MGT, 1), Ch);
1192 "Operand types must be vectors");
1207 void DAGTypeLegalizer::SplitVecRes_UnaryOp(
SDNode *N,
SDValue &Lo,
1231 void DAGTypeLegalizer::SplitVecRes_ExtendOp(
SDNode *N,
SDValue &Lo,
1253 if ((NumElements & 1) == 0 &&
1262 EVT SplitLoVT, SplitHiVT;
1266 DEBUG(
dbgs() <<
"Split vector extend via incremental extend:";
1280 SplitVecRes_UnaryOp(N, Lo, Hi);
1288 GetSplitVector(N->
getOperand(0), Inputs[0], Inputs[1]);
1289 GetSplitVector(N->
getOperand(1), Inputs[2], Inputs[3]);
1304 unsigned InputUsed[2] = { -1U, -1U };
1305 unsigned FirstMaskIdx =
High * NewElts;
1306 bool useBuildVector =
false;
1307 for (
unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1309 int Idx = N->
getMaskElt(FirstMaskIdx + MaskOffset);
1312 unsigned Input = (
unsigned)Idx / NewElts;
1321 Idx -= Input * NewElts;
1326 if (InputUsed[OpNo] == Input) {
1329 }
else if (InputUsed[OpNo] == -1U) {
1331 InputUsed[OpNo] = Input;
1339 useBuildVector =
true;
1347 if (useBuildVector) {
1352 for (
unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
1354 int Idx = N->
getMaskElt(FirstMaskIdx + MaskOffset);
1357 unsigned Input = (
unsigned)Idx / NewElts;
1366 Idx -= Input * NewElts;
1376 }
else if (InputUsed[0] == -1U) {
1380 SDValue Op0 = Inputs[InputUsed[0]];
1382 SDValue Op1 = InputUsed[1] == -1U ?
1383 DAG.
getUNDEF(NewVT) : Inputs[InputUsed[1]];
1401 bool DAGTypeLegalizer::SplitVectorOperand(
SDNode *N,
unsigned OpNo) {
1411 if (!Res.getNode()) {
1415 dbgs() <<
"SplitVectorOperand Op #" << OpNo <<
": ";
1422 case ISD::SETCC: Res = SplitVecOp_VSETCC(N);
break;
1428 Res = SplitVecOp_TruncateHelper(N);
1433 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1436 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1439 Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1442 Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1445 Res = SplitVecOp_VSELECT(N, OpNo);
1450 Res = SplitVecOp_TruncateHelper(N);
1452 Res = SplitVecOp_UnaryOp(N);
1457 Res = SplitVecOp_TruncateHelper(N);
1459 Res = SplitVecOp_UnaryOp(N);
1469 Res = SplitVecOp_UnaryOp(N);
1475 if (!Res.getNode())
return false;
1479 if (Res.getNode() ==
N)
1483 "Invalid operand expansion");
1485 ReplaceValueWith(
SDValue(N, 0), Res);
1489 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(
SDNode *N,
unsigned OpNo) {
1492 assert(OpNo == 0 &&
"Illegal operand must be mask");
1503 assert(Lo.getValueType() == Hi.getValueType() &&
1504 "Lo and Hi have differing types");
1508 assert(LoOpVT == HiOpVT &&
"Asymmetric vector split?");
1510 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
1511 std::tie(LoOp0, HiOp0) = DAG.
SplitVector(Src0, DL);
1512 std::tie(LoOp1, HiOp1) = DAG.
SplitVector(Src1, DL);
1513 std::tie(LoMask, HiMask) = DAG.
SplitVector(Mask, DL);
1546 Lo = BitConvertToInteger(Lo);
1547 Hi = BitConvertToInteger(Hi);
1553 JoinIntegers(Lo, Hi));
1556 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(
SDNode *N) {
1565 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1567 if (IdxVal < LoElts) {
1569 "Extracted subvector crosses vector split!");
1578 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(
SDNode *N) {
1583 if (isa<ConstantSDNode>(Idx)) {
1584 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1588 GetSplitVector(Vec, Lo, Hi);
1590 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1592 if (IdxVal < LoElts)
1648 GetSplitVector(Mask, MaskLo, MaskHi);
1650 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, dl);
1653 EVT LoMemVT, HiMemVT;
1658 GetSplitVector(Src0, Src0Lo, Src0Hi);
1660 std::tie(Src0Lo, Src0Hi) = DAG.
SplitVector(Src0, dl);
1664 GetSplitVector(Index, IndexLo, IndexHi);
1666 std::tie(IndexLo, IndexHi) = DAG.
SplitVector(Index, dl);
1673 SDValue OpsLo[] = {Ch, Src0Lo, MaskLo,
Ptr, IndexLo};
1683 SDValue OpsHi[] = {Ch, Src0Hi, MaskHi,
Ptr, IndexHi};
1694 ReplaceValueWith(
SDValue(MGT, 1), Ch);
1698 ReplaceValueWith(
SDValue(MGT, 0), Res);
1712 EVT LoMemVT, HiMemVT;
1718 GetSplitVector(Data, DataLo, DataHi);
1720 std::tie(DataLo, DataHi) = DAG.
SplitVector(Data, DL);
1725 GetSplitVector(Mask, MaskLo, MaskHi);
1727 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, DL);
1729 MaskLo = PromoteTargetBoolean(MaskLo, DataLo.
getValueType());
1730 MaskHi = PromoteTargetBoolean(MaskHi, DataHi.
getValueType());
1734 unsigned SecondHalfAlignment =
1736 Alignment/2 : Alignment;
1744 Lo = DAG.
getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
1755 Hi = DAG.
getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
1775 EVT LoMemVT, HiMemVT;
1781 GetSplitVector(Data, DataLo, DataHi);
1783 std::tie(DataLo, DataHi) = DAG.
SplitVector(Data, DL);
1788 GetSplitVector(Mask, MaskLo, MaskHi);
1790 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, DL);
1794 GetSplitVector(Index, IndexLo, IndexHi);
1796 std::tie(IndexLo, IndexHi) = DAG.
SplitVector(Index, DL);
1804 SDValue OpsLo[] = {Ch, DataLo, MaskLo,
Ptr, IndexLo};
1813 SDValue OpsHi[] = {Ch, DataHi, MaskHi,
Ptr, IndexHi};
1824 assert(OpNo == 1 &&
"Can only split the stored value");
1837 EVT LoMemVT, HiMemVT;
1840 unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1844 Alignment, MMOFlags, AAInfo);
1856 HiMemVT, Alignment, MMOFlags, AAInfo);
1860 Alignment, MMOFlags, AAInfo);
1865 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(
SDNode *N) {
1876 for (
unsigned i = 0, e = Op.getValueType().getVectorNumElements();
1887 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(
SDNode *N) {
1913 assert(!(NumElements & 1) &&
"Splitting vector, but not in half!");
1921 if (InElementSize <= OutElementSize * 2)
1922 return SplitVecOp_UnaryOp(N);
1927 std::tie(InLoVec, InHiVec) = DAG.
SplitVector(InVec, DL);
1929 EVT HalfElementVT = IsFloat ?
1931 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
1954 "Operand types must be vectors");
1956 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
1960 unsigned PartElements = Lo0.getValueType().getVectorNumElements();
1999 void DAGTypeLegalizer::WidenVectorResult(
SDNode *N,
unsigned ResNo) {
2000 DEBUG(
dbgs() <<
"Widen node result " << ResNo <<
": ";
2012 dbgs() <<
"WidenVectorResult #" << ResNo <<
": ";
2019 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N);
break;
2025 case ISD::LOAD: Res = WidenVecRes_LOAD(N);
break;
2029 case ISD::SELECT: Res = WidenVecRes_SELECT(N);
break;
2031 case ISD::SETCC: Res = WidenVecRes_SETCC(N);
break;
2032 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N);
break;
2034 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2037 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2040 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2059 Res = WidenVecRes_Binary(N);
2072 Res = WidenVecRes_BinaryCanTrap(N);
2076 Res = WidenVecRes_FCOPYSIGN(N);
2080 Res = WidenVecRes_POWI(N);
2086 Res = WidenVecRes_Shift(N);
2092 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2105 Res = WidenVecRes_Convert(N);
2129 Res = WidenVecRes_Unary(N);
2132 Res = WidenVecRes_Ternary(N);
2138 SetWidenedVector(
SDValue(N, ResNo), Res);
2160 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(
SDNode *N) {
2170 NumElts = NumElts / 2;
2192 unsigned ConcatEnd = 0;
2200 while (CurNumElts != 0) {
2201 while (CurNumElts >= NumElts) {
2208 ConcatOps[ConcatEnd++] = DAG.
getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
2210 CurNumElts -= NumElts;
2213 NumElts = NumElts / 2;
2218 for (
unsigned i = 0;
i != CurNumElts; ++
i, ++Idx) {
2225 ConcatOps[ConcatEnd++] = DAG.
getNode(Opcode, dl, WidenEltVT,
2233 if (ConcatEnd == 1) {
2234 VT = ConcatOps[0].getValueType();
2236 return ConcatOps[0];
2243 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2244 Idx = ConcatEnd - 1;
2245 VT = ConcatOps[Idx--].getValueType();
2246 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
2259 unsigned NumToInsert = ConcatEnd - Idx - 1;
2260 for (
unsigned i = 0, OpIdx = Idx+1;
i < NumToInsert;
i++, OpIdx++) {
2265 ConcatOps[Idx+1] = VecOp;
2266 ConcatEnd = Idx + 2;
2272 unsigned RealVals = ConcatEnd - Idx - 1;
2273 unsigned SubConcatEnd = 0;
2274 unsigned SubConcatIdx = Idx + 1;
2275 while (SubConcatEnd < RealVals)
2276 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2277 while (SubConcatEnd < OpsToConcat)
2278 SubConcatOps[SubConcatEnd++] = undefVec;
2280 NextVT, SubConcatOps);
2281 ConcatEnd = SubConcatIdx + 1;
2286 if (ConcatEnd == 1) {
2287 VT = ConcatOps[0].getValueType();
2289 return ConcatOps[0];
2294 if (NumOps != ConcatEnd ) {
2296 for (
unsigned j = ConcatEnd; j < NumOps; ++j)
2297 ConcatOps[j] = UndefVal;
2320 InVTNumElts = InVT.getVectorNumElements();
2321 if (InVTNumElts == WidenNumElts) {
2323 return DAG.
getNode(Opcode, DL, WidenVT, InOp);
2334 if (WidenNumElts % InVTNumElts == 0) {
2336 unsigned NumConcat = WidenNumElts/InVTNumElts;
2340 for (
unsigned i = 1;
i != NumConcat; ++
i)
2344 return DAG.
getNode(Opcode, DL, WidenVT, InVec);
2348 if (InVTNumElts % WidenNumElts == 0) {
2354 return DAG.
getNode(Opcode, DL, WidenVT, InVal);
2362 unsigned MinElts =
std::min(InVTNumElts, WidenNumElts);
2364 for (i=0; i < MinElts; ++
i) {
2369 Ops[i] = DAG.
getNode(Opcode, DL, EltVT, Val);
2375 for (; i < WidenNumElts; ++
i)
2381 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(
SDNode *N) {
2395 InOp = GetWidenedVector(InOp);
2411 for (
unsigned i = 0, e =
std::min(InVTNumElts, WidenNumElts); i != e; ++
i) {
2430 while (Ops.
size() != WidenNumElts)
2436 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(
SDNode *N) {
2440 return WidenVecRes_BinaryCanTrap(N);
2459 EVT ShVT = ShOp.getValueType();
2461 ShOp = GetWidenedVector(ShOp);
2462 ShVT = ShOp.getValueType();
2467 if (ShVT != ShWidenVT)
2468 ShOp = ModifyToType(ShOp, ShWidenVT);
2484 .getVectorElementType(),
2491 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(
SDNode *N,
unsigned ResNo) {
2492 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
2493 return GetWidenedVector(WidenVec);
2503 switch (getTypeAction(InVT)) {
2515 InOp = GetPromotedInteger(InOp);
2517 if (WidenVT.
bitsEq(InVT))
2530 InOp = GetWidenedVector(InOp);
2532 if (WidenVT.
bitsEq(InVT))
2541 if (WidenSize % InSize == 0 && InVT !=
MVT::x86mmx) {
2546 unsigned NewNumElts = WidenSize / InSize;
2564 for (
unsigned i = 1; i < NewNumElts; ++
i)
2576 return CreateStackStoreLoad(InOp, WidenVT);
2579 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(
SDNode *N) {
2593 assert(WidenNumElts >= NumElts &&
"Shrinking vector instead of widening!");
2594 NewOps.append(WidenNumElts - NumElts, DAG.
getUNDEF(EltVT));
2599 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(
SDNode *N) {
2607 bool InputWidened =
false;
2615 for (
unsigned i=0; i < NumOperands; ++
i)
2617 for (
unsigned i = NumOperands; i != NumConcat; ++
i)
2622 InputWidened =
true;
2626 for (i=1; i < NumOperands; ++
i)
2630 if (i == NumOperands)
2635 if (NumOperands == 2) {
2638 for (
unsigned i = 0; i < NumInElts; ++
i) {
2640 MaskOps[i + NumInElts] = i + WidenNumElts;
2654 for (
unsigned i=0; i < NumOperands; ++
i) {
2657 InOp = GetWidenedVector(InOp);
2658 for (
unsigned j=0; j < NumInElts; ++j)
2664 for (; Idx < WidenNumElts; ++Idx)
2665 Ops[Idx] = UndefVal;
2669 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(
SDNode *N) {
2678 InOp = GetWidenedVector(InOp);
2683 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2684 if (IdxVal == 0 && InVT == WidenVT)
2689 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
2698 for (i=0; i < NumElts; ++
i)
2705 for (; i < WidenNumElts; ++
i)
2710 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(
SDNode *N) {
2724 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
2726 Result = GenWidenVectorLoads(LdChain, LD);
2732 if (LdChain.
size() == 1)
2733 NewChain = LdChain[0];
2739 ReplaceValueWith(
SDValue(N, 1), NewChain);
2754 Mask = GetWidenedVector(Mask);
2756 EVT BoolVT = getSetCCResultType(WidenVT);
2763 unsigned NumConcat = WidenNumElts / MaskNumElts;
2767 for (
unsigned i = 1; i != NumConcat; ++
i)
2792 Mask = WidenTargetBoolean(Mask, WideVT,
true);
2799 Index = ModifyToType(Index, WideIndexVT);
2811 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(
SDNode *N) {
2826 CondEltVT, WidenNumElts);
2828 Cond1 = GetWidenedVector(Cond1);
2836 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
2837 SDValue Res = ModifyToType(SplitSelect, WidenVT);
2842 Cond1 = ModifyToType(Cond1, CondWidenVT);
2849 WidenVT, Cond1, InOp1, InOp2);
2852 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(
SDNode *N) {
2863 "Scalar/Vector type mismatch");
2891 for (
unsigned i = 0; i != NumElts; ++
i) {
2893 if (Idx < (
int)NumElts)
2894 NewMask.push_back(Idx);
2896 NewMask.push_back(Idx - NumElts + WidenNumElts);
2898 for (
unsigned i = NumElts; i != WidenNumElts; ++
i)
2899 NewMask.push_back(-1);
2906 "Operands must be vectors");
2908 unsigned WidenNumElts = WidenVT.getVectorNumElements();
2920 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
2921 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
2925 InOp1 = GetWidenedVector(InOp1);
2932 "Input not widened to expected type!");
2942 bool DAGTypeLegalizer::WidenVectorOperand(
SDNode *N,
unsigned OpNo) {
2943 DEBUG(
dbgs() <<
"Widen node operand " << OpNo <<
": ";
2955 dbgs() <<
"WidenVectorOperand op #" << OpNo <<
": ";
2965 case ISD::STORE: Res = WidenVecOp_STORE(N);
break;
2966 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo);
break;
2967 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo);
break;
2968 case ISD::SETCC: Res = WidenVecOp_SETCC(N);
break;
2974 Res = WidenVecOp_EXTEND(N);
2983 Res = WidenVecOp_Convert(N);
2988 if (!Res.getNode())
return false;
2992 if (Res.getNode() ==
N)
2997 "Invalid operand expansion");
2999 ReplaceValueWith(
SDValue(N, 0), Res);
3012 return WidenVecOp_Convert(N);
3013 InOp = GetWidenedVector(InOp);
3016 "Input wasn't widened!");
3028 FixedEltVT == InEltVT) {
3030 "Not enough elements in the fixed type for the operand!");
3032 "We can't have the same type as we started with!");
3049 return WidenVecOp_Convert(N);
3083 InOp = GetWidenedVector(InOp);
3089 for (
unsigned i=0; i < NumElts; ++
i)
3106 unsigned InWidenSize = InWidenVT.getSizeInBits();
3110 unsigned NewNumElts = InWidenSize / Size;
3120 return CreateStackStoreLoad(InOp, VT);
3123 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(
SDNode *N) {
3138 for (
unsigned i=0; i < NumOperands; ++
i) {
3141 InOp = GetWidenedVector(InOp);
3142 for (
unsigned j=0; j < NumInElts; ++j)
3150 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(
SDNode *N) {
3156 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(
SDNode *N) {
3169 GenWidenVectorTruncStores(StChain, ST);
3171 GenWidenVectorStores(StChain, ST);
3173 if (StChain.size() == 1)
3179 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(
SDNode *N,
unsigned OpNo) {
3182 EVT MaskVT = Mask.getValueType();
3185 SDValue WideVal = GetWidenedVector(StVal);
3189 Mask = GetWidenedVector(Mask);
3198 unsigned NumConcat = WidenNumElts / MaskNumElts;
3202 for (
unsigned i = 1; i != NumConcat; ++
i)
3207 assert(Mask.getValueType().getVectorNumElements() ==
3209 "Mask and data vectors should have the same number of elements");
3215 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(
SDNode *N,
unsigned OpNo) {
3216 assert(OpNo == 1 &&
"Can widen only data operand of mscatter");
3222 SDValue WideVal = GetWidenedVector(DataOp);
3228 Mask = WidenTargetBoolean(Mask, WideVT,
true);
3235 Index = ModifyToType(Index, WideIndexVT);
3283 unsigned Width,
EVT WidenVT,
3284 unsigned Align = 0,
unsigned WidenEx = 0) {
3288 unsigned AlignInBits = Align*8;
3291 EVT RetVT = WidenEltVT;
3292 if (Width == WidenEltWidth)
3306 (WidenWidth % MemVTWidth) == 0 &&
3308 (MemVTWidth <= Width ||
3309 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3322 (WidenWidth % MemVTWidth) == 0 &&
3324 (MemVTWidth <= Width ||
3325 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
3340 unsigned Start,
unsigned End) {
3342 SDLoc dl(LdOps[Start]);
3343 EVT LdTy = LdOps[Start].getValueType();
3351 for (
unsigned i = Start + 1; i !=
End; ++
i) {
3352 EVT NewLdTy = LdOps[
i].getValueType();
3353 if (NewLdTy != LdTy) {
3389 int WidthDiff = WidenWidth - LdWidth;
3390 unsigned LdAlign = LD->
isVolatile() ? 0 : Align;
3393 EVT NewVT =
FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3396 Align, MMOFlags, AAInfo);
3400 if (LdWidth <= NewVTWidth) {
3402 unsigned NumElts = WidenWidth / NewVTWidth;
3407 if (NewVT == WidenVT)
3410 assert(WidenWidth % NewVTWidth == 0);
3411 unsigned NumConcat = WidenWidth / NewVTWidth;
3414 ConcatOps[0] = LdOp;
3415 for (
unsigned i = 1; i != NumConcat; ++
i)
3416 ConcatOps[i] = UndefVal;
3424 LdWidth -= NewVTWidth;
3425 unsigned Offset = 0;
3427 while (LdWidth > 0) {
3428 unsigned Increment = NewVTWidth / 8;
3429 Offset += Increment;
3434 if (LdWidth < NewVTWidth) {
3436 NewVT =
FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
3438 L = DAG.
getLoad(NewVT, dl, Chain, BasePtr,
3440 MinAlign(Align, Increment), MMOFlags, AAInfo);
3449 while (size < LdOp->getValueSizeInBits(0)) {
3456 L = DAG.
getLoad(NewVT, dl, Chain, BasePtr,
3458 MinAlign(Align, Increment), MMOFlags, AAInfo);
3465 LdWidth -= NewVTWidth;
3470 if (!LdOps[0].getValueType().
isVector())
3480 EVT LdTy = LdOps[
i].getValueType();
3483 for (--i; i >= 0; --
i) {
3484 LdTy = LdOps[
i].getValueType();
3490 ConcatOps[--Idx] = LdOps[
i];
3491 for (--i; i >= 0; --
i) {
3492 EVT NewLdTy = LdOps[
i].getValueType();
3493 if (NewLdTy != LdTy) {
3500 ConcatOps[--Idx] = LdOps[
i];
3513 for (; i != End-Idx; ++
i)
3514 WidenOps[i] = ConcatOps[Idx+i];
3515 for (; i != NumOps; ++
i)
3516 WidenOps[i] = UndefVal;
3549 LdEltVT, Align, MMOFlags, AAInfo);
3551 unsigned i = 0, Offset = Increment;
3552 for (i=1; i < NumElts; ++
i, Offset += Increment) {
3557 Ops[
i] = DAG.
getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
3559 Align, MMOFlags, AAInfo);
3565 for (; i != WidenNumElts; ++
i)
3594 unsigned Offset = 0;
3595 while (StWidth != 0) {
3599 unsigned Increment = NewVTWidth / 8;
3608 MinAlign(Align, Offset), MMOFlags, AAInfo));
3609 StWidth -= NewVTWidth;
3610 Offset += Increment;
3615 }
while (StWidth != 0 && StWidth >= NewVTWidth);
3618 unsigned NumElts = ValWidth / NewVTWidth;
3622 Idx = Idx * ValEltWidth / NewVTWidth;
3630 MinAlign(Align, Offset), MMOFlags, AAInfo));
3631 StWidth -= NewVTWidth;
3632 Offset += Increment;
3636 }
while (StWidth != 0 && StWidth >= NewVTWidth);
3638 Idx = Idx * NewVTWidth / ValEltWidth;
3676 unsigned Offset = Increment;
3677 for (
unsigned i=1; i < NumElts; ++
i, Offset += Increment) {
3687 StEltVT,
MinAlign(Align, Offset), MMOFlags, AAInfo));
3695 bool FillWithZeroes) {
3700 "input and widen element type must match");
3709 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
3710 unsigned NumConcat = WidenNumElts / InNumElts;
3715 for (
unsigned i = 1; i != NumConcat; ++
i)
3721 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
3729 unsigned MinNumElts =
std::min(WidenNumElts, InNumElts);
3731 for (Idx = 0; Idx < MinNumElts; ++Idx)
3738 for ( ; Idx < WidenNumElts; ++Idx)
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
unsigned getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
void push_back(const T &Elt)
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
SDValue getValue(unsigned R) const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
Flags getFlags() const
Return the raw flags of the source value,.
LLVMContext * getContext() const
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
SDValue getSignExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will sign extend the low lanes of the operand into the specified vector typ...
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
static ISD::NodeType getExtendForContent(BooleanContent Content)
const SDValue & getSrc0() const
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned Num) const
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const SDValue & getBasePtr() const
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
static SDValue BuildVectorFromScalar(SelectionDAG &DAG, EVT VecTy, SmallVectorImpl< SDValue > &LdOps, unsigned Start, unsigned End)
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will any-extend the low lanes of the operand into the specified vector type...
bool isVector() const
isVector - Return true if this is a vector value type.
virtual bool canOpTrap(unsigned Op, EVT VT) const
Returns true if the operation can trap for the value type.
A description of a memory reference used in the backend.
struct fuzzer::@269 Flags
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Shift and rotation operations.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
MachineFunction & getMachineFunction() const
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
MachinePointerInfo getWithOffset(int64_t O) const
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const
Increments memory address Addr according to the type of the value DataVT that should be stored...
int getMaskElt(unsigned Idx) const
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
ISD::LoadExtType getExtensionType() const
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Simple integer binary arithmetic operators.
const SDValue & getBasePtr() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
EVT getMemoryVT() const
Return the type of the in-memory value.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Idx) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
const DataLayout & getDataLayout() const
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
UNDEF - An undefined node.
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
SDNode * getNode() const
get the SDNode which holds the desired result
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
unsigned getScalarSizeInBits() const
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
constexpr bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
static EVT FindMemType(SelectionDAG &DAG, const TargetLowering &TLI, unsigned Width, EVT WidenVT, unsigned Align=0, unsigned WidenEx=0)
const SDValue & getOperand(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
Simple binary floating point operators.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
unsigned getOriginalAlignment() const
Returns alignment and volatility of the memory access.
static EVT getFloatingPointVT(unsigned BitWidth)
getFloatingPointVT - Returns the EVT that represents a floating point type with the given number of b...
static const unsigned End
const SDValue & getBasePtr() const
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
const SDValue & getValue() const
Bit counting operators with an undefined result for zero inputs.
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
EVT - Extended Value Type.
const SDValue & getMask() const
bool bitsEq(EVT VT) const
bitsEq - Return true if this has the same number of bits as VT.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
This class contains a discriminated union of information about pointers in memory operands...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
const MachinePointerInfo & getPointerInfo() const
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
The memory access writes data.
SDValue getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return an operation which will zero extend the low lanes of the operand into the specified vector typ...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
void dump() const
Dump this node, for debugging.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part...
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool isVector(MCInstrInfo const &MCII, MCInst const &MCI)
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
const SDValue & getChain() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Byte Swap and Counting operators.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
const SDValue & getValue() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Select(COND, TRUEVAL, FALSEVAL).
op_iterator op_begin() const
ZERO_EXTEND - Used for integer types, zeroing the new bits.
ANY_EXTEND - Used for integer types. The high bits are undefined.
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
FMINNAN/FMAXNAN - Behave identically to FMINNUM/FMAXNUM, except that when a single input is NaN...
iterator_range< value_op_iterator > op_values() const
Flags
Flags values. These may be or'd together.
const SDValue & getMask() const
The memory access reads data.
These are IR-level optimization flags that may be propagated to SDNodes.
Bitwise operators - logical and, logical or, logical xor.
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
op_iterator op_end() const
bool isExpandingLoad() const
This class is used to represent an MSCATTER node.
const SDValue & getIndex() const
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This class is used to represent an MLOAD node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
FMA - Perform a * b + c with no intermediate rounding step.
const SDValue & getBasePtr() const
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
const TargetLowering & getTargetLoweringInfo() const
SetCC operator - This evaluates to a true value iff the condition is true.
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
BooleanContent
Enum that describes how the target represents true/false values.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
TRUNCATE - Completely drop the high bits.
unsigned getAlignment() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
const SDNodeFlags * getFlags() const
This could be defined as a virtual function and implemented more simply and directly, but it is not to avoid creating a vtable for this class.
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
This class is used to represent ISD::LOAD nodes.