32#define DEBUG_TYPE "legalize-types"
42void DAGTypeLegalizer::PromoteIntegerResult(
SDNode *
N,
unsigned ResNo) {
47 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
52 switch (
N->getOpcode()) {
55 dbgs() <<
"PromoteIntegerResult #" << ResNo <<
": ";
56 N->dump(&DAG);
dbgs() <<
"\n";
64 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(
N);
break;
68 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(
N);
break;
69 case ISD::CTLS: Res = PromoteIntRes_CTLS(
N);
break;
71 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(
N);
break;
73 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(
N);
break;
76 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
77 case ISD::VP_CTTZ_ELTS:
78 Res = PromoteIntRes_VP_CttzElements(
N);
81 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(
N);
break;
91 Res = PromoteIntRes_VECTOR_COMPRESS(
N);
94 Res = PromoteIntRes_VECTOR_SHUFFLE_VAR(
N);
99 Res = PromoteIntRes_Select(
N);
104 case ISD::SETCC: Res = PromoteIntRes_SETCC(
N);
break;
106 case ISD::SHL: Res = PromoteIntRes_SHL(
N);
break;
108 Res = PromoteIntRes_SIGN_EXTEND_INREG(
N);
break;
109 case ISD::SRA: Res = PromoteIntRes_SRA(
N);
break;
110 case ISD::SRL: Res = PromoteIntRes_SRL(
N);
break;
113 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(
N);
break;
114 case ISD::VAARG: Res = PromoteIntRes_VAARG(
N);
break;
118 Res = PromoteIntRes_EXTRACT_SUBVECTOR(
N);
break;
120 Res = PromoteIntRes_INSERT_SUBVECTOR(
N);
break;
122 Res = PromoteIntRes_VECTOR_REVERSE(
N);
break;
124 Res = PromoteIntRes_VECTOR_SHUFFLE(
N);
break;
127 Res = PromoteIntRes_VECTOR_SPLICE(
N);
129 case ISD::EXPERIMENTAL_VP_SPLICE:
130 Res = PromoteIntRes_VP_SPLICE(
N);
133 Res = PromoteIntRes_VECTOR_REPEAT(
N);
137 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
N);
140 Res = PromoteIntRes_INSERT_VECTOR_ELT(
N);
break;
142 Res = PromoteIntRes_BUILD_VECTOR(
N);
146 Res = PromoteIntRes_ScalarOp(
N);
150 Res = PromoteIntRes_CONCAT_VECTORS(
N);
break;
155 Res = PromoteIntRes_EXTEND_VECTOR_INREG(
N);
break;
158 Res = PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(
N);
162 Res = PromoteIntRes_GET_ACTIVE_LANE_MASK(
N);
165 Res = PromoteIntRes_MASK_BEFOREFIRST(
N);
168 Res = PromoteIntRes_VECTOR_MATCH(
N);
174 Res = PromoteIntRes_PARTIAL_REDUCE_MLA(
N);
188 Res = PromoteIntRes_FP_TO_XINT_SAT(
N);
break;
192 Res = PromoteIntRes_FP_TO_FP16_BF16(
N);
195 Res = PromoteIntRes_CONVERT_TO_ARBITRARY_FP(
N);
199 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(
N);
208 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(
N);
break;
218 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
225 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(
N);
break;
229 case ISD::ABDU: Res = PromoteIntRes_SExtOrZExtIntBinOp(
N);
break;
233 Res = PromoteIntRes_ZExtMaskedIntBinOp(
N);
237 Res = PromoteIntRes_SExtMaskedIntBinOp(
N);
241 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(
N, ResNo);
break;
243 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(
N, ResNo);
break;
245 case ISD::UMULO: Res = PromoteIntRes_XMULO(
N, ResNo);
break;
261 Res = PromoteIntRes_ADDSUBSHLSAT(
N);
266 Res = PromoteIntRes_CMP(
N);
281 Res = PromoteIntRes_ABS(
N);
315 Res = PromoteIntRes_VECREDUCE(
N);
318 case ISD::VP_REDUCE_ADD:
319 case ISD::VP_REDUCE_MUL:
320 case ISD::VP_REDUCE_AND:
321 case ISD::VP_REDUCE_OR:
322 case ISD::VP_REDUCE_XOR:
323 case ISD::VP_REDUCE_SMAX:
324 case ISD::VP_REDUCE_SMIN:
325 case ISD::VP_REDUCE_UMAX:
326 case ISD::VP_REDUCE_UMIN:
327 Res = PromoteIntRes_VP_REDUCE(
N);
332 Res = PromoteIntRes_LOOP_DEPENDENCE_MASK(
N);
336 Res = PromoteIntRes_FREEZE(
N);
341 Res = PromoteIntRes_Rotate(
N);
346 Res = PromoteIntRes_FunnelShift(
N);
352 Res = PromoteIntRes_CLMUL(
N);
356 Res = PromoteIntRes_PEXT(
N);
360 Res = PromoteIntRes_PDEP(
N);
365 Res = PromoteIntRes_MULH(
N);
369 Res = PromoteIntRes_IS_FPCLASS(
N);
372 Res = PromoteIntRes_FFREXP(
N);
377 Res = PromoteIntRes_XRINT(
N);
381 Res = PromoteIntRes_PATCHPOINT(
N);
384 Res = PromoteIntRes_READ_REGISTER(
N);
390 SetPromotedInteger(
SDValue(
N, ResNo), Res);
395 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
396 return GetPromotedInteger(
Op);
399SDValue DAGTypeLegalizer::PromoteIntRes_LOOP_DEPENDENCE_MASK(
SDNode *
N) {
401 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
402 return DAG.getNode(
N->getOpcode(), SDLoc(
N), NewVT,
N->ops());
407 SDValue Op = SExtPromotedInteger(
N->getOperand(0));
409 Op.getValueType(),
Op,
N->getOperand(1));
414 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
416 Op.getValueType(),
Op,
N->getOperand(1));
420 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
423 switch (TLI.getExtendForAtomicOps()) {
439 DAG.getAtomicLoad(ExtType, SDLoc(
N),
N->getMemoryVT(), ResVT,
440 N->getChain(),
N->getBasePtr(),
N->getMemOperand());
450 switch (TLI.getExtendForAtomicRMWArg(
N->getOpcode())) {
452 Op2 = SExtPromotedInteger(Op2);
455 Op2 = ZExtPromotedInteger(Op2);
458 Op2 = GetPromotedInteger(Op2);
463 SDValue Res = DAG.getAtomic(
N->getOpcode(), SDLoc(
N),
465 N->getChain(),
N->getBasePtr(),
466 Op2,
N->getMemOperand());
477 EVT SVT = getSetCCResultType(
N->getOperand(2).getValueType());
478 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(1));
482 if (!TLI.isTypeLegal(SVT))
485 SDVTList VTs = DAG.getVTList(
N->getValueType(0), SVT, MVT::Other);
486 SDValue Res = DAG.getAtomicCmpSwap(
488 N->getChain(),
N->getBasePtr(),
N->getOperand(2),
N->getOperand(3),
492 return DAG.getSExtOrTrunc(Res.
getValue(1), SDLoc(
N), NVT);
498 SDValue Op3 = GetPromotedInteger(
N->getOperand(3));
499 switch (TLI.getExtendForAtomicCmpSwapArg()) {
501 Op2 = SExtPromotedInteger(Op2);
504 Op2 = ZExtPromotedInteger(Op2);
507 Op2 = GetPromotedInteger(Op2);
514 DAG.getVTList(Op2.
getValueType(),
N->getValueType(1), MVT::Other);
515 SDValue Res = DAG.getAtomicCmpSwap(
516 N->getOpcode(), SDLoc(
N),
N->getMemoryVT(), VTs,
N->getChain(),
517 N->getBasePtr(), Op2, Op3,
N->getMemOperand());
519 for (
unsigned i = 1, NumResults =
N->getNumValues(); i < NumResults; ++i)
527 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
528 EVT OutVT =
N->getValueType(0);
529 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
532 switch (getTypeAction(InVT)) {
538 return DAG.getNode(
ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
542 return DAG.getNode(
ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
545 return DAG.getNode(
ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
553 BitConvertToInteger(GetScalarizedVector(InOp)));
562 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
563 Lo = BitConvertToInteger(
Lo);
564 Hi = BitConvertToInteger(
Hi);
566 if (DAG.getDataLayout().isBigEndian())
572 JoinIntegers(
Lo,
Hi));
587 if (DAG.getDataLayout().isBigEndian()) {
591 DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl));
606 if (isTypeLegal(WideOutVT)) {
607 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
609 DAG.getVectorIdxConstant(0, dl));
618 DAG.getDataLayout().isLittleEndian()) {
629 if (isTypeLegal(WideVecVT)) {
631 DAG.getUNDEF(WideVecVT), InOp,
632 DAG.getVectorIdxConstant(0, dl));
640 CreateStackStoreLoad(InOp, OutVT));
644 SDValue V = GetPromotedInteger(
N->getOperand(0));
646 V.getValueType(),
V);
650 SDValue Op = GetPromotedInteger(
N->getOperand(0));
651 EVT OVT =
N->getValueType(0);
652 EVT NVT =
Op.getValueType();
660 !TLI.isOperationLegalOrCustomOrPromote(
ISD::BSWAP, NVT)) {
661 if (
SDValue Res = TLI.expandBSWAP(
N, DAG))
666 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
672 SDValue Op = GetPromotedInteger(
N->getOperand(0));
673 EVT OVT =
N->getValueType(0);
674 EVT NVT =
Op.getValueType();
683 if (
SDValue Res = TLI.expandBITREVERSE(
N, DAG))
688 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
697 TLI.getTypeToTransformTo(*DAG.getContext(),
698 N->getValueType(0)), JoinIntegers(
N->getOperand(0),
703 EVT VT =
N->getValueType(0);
710 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
717 EVT OVT =
N->getValueType(0);
718 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
724 if (!OVT.
isVector() && TLI.isTypeLegal(NVT) &&
725 !TLI.isOperationLegalOrCustomOrPromote(
ISD::CTLZ, NVT) &&
727 if (
SDValue Result = TLI.expandCTLZ(
N, DAG)) {
733 unsigned CtlzOpcode =
N->getOpcode();
736 SDValue ExtractLeadingBits = DAG.getConstant(
739 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
743 return DAG.getNode(
ISD::SUB, dl, NVT,
744 DAG.getNode(
N->getOpcode(), dl, NVT,
Op),
749 SDValue Op = GetPromotedInteger(
N->getOperand(0));
753 DAG.getShiftAmountConstant(SHLAmount,
Op.getValueType(), dl);
755 return DAG.getNode(CtlzOpcode, dl, NVT,
Op);
761 EVT OVT =
N->getValueType(0);
762 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
765 SDValue ExtractLeadingBits = DAG.getConstant(
768 SDValue Op = SExtPromotedInteger(
N->getOperand(0));
774 EVT OVT =
N->getValueType(0);
775 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
783 !TLI.isOperationLegalOrCustomOrPromote(
ISD::CTPOP, NVT)) {
784 if (
SDValue Result = TLI.expandCTPOP(
N, DAG)) {
791 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
792 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
Op.getValueType(),
Op);
796 SDValue Op = GetPromotedInteger(
N->getOperand(0));
797 EVT OVT =
N->getValueType(0);
798 EVT NVT =
Op.getValueType();
805 if (!OVT.
isVector() && TLI.isTypeLegal(NVT) &&
806 !TLI.isOperationLegalOrCustomOrPromote(
ISD::CTTZ, NVT) &&
810 if (
SDValue Result = TLI.expandCTTZ(
N, DAG)) {
816 unsigned NewOpc =
N->getOpcode();
823 Op = DAG.getNode(
ISD::OR, dl, NVT,
Op, DAG.getConstant(TopBit, dl, NVT));
826 return DAG.getNode(NewOpc, dl, NVT,
Op);
829SDValue DAGTypeLegalizer::PromoteIntRes_VP_CttzElements(
SDNode *
N) {
831 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
832 return DAG.getNode(
N->getOpcode(),
DL, NewVT,
N->ops());
835SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
837 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
844 if (TLI.getTypeAction(*DAG.getContext(), Op0.
getValueType())
850 EVT SVT =
In.getValueType().getScalarType();
853 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
861 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
863 TLI.getPreferredFPToIntOpcode(
N->getOpcode(),
N->getValueType(0), NVT);
867 if (
N->isStrictFPOpcode()) {
868 Res = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
869 {
N->getOperand(0),
N->getOperand(1)});
874 Res = DAG.
getNode(NewOpc, dl, NVT,
N->getOperand(0));
889 DAG.getValueType(
N->getValueType(0).getScalarType()));
892SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(
SDNode *
N) {
894 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
896 return DAG.getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
900SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(
SDNode *
N) {
901 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
904 return DAG.getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
909SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_TO_ARBITRARY_FP(
SDNode *
N) {
910 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
914 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
917SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(
SDNode *
N) {
918 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
921 SDValue Res = DAG.
getNode(
N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
922 N->getOperand(0),
N->getOperand(1));
928 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
930 return DAG.getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
934 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
938 DAG.
getNode(
N->getOpcode(), dl, {NVT, MVT::Other},
N->getOperand(0));
947 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
950 if (getTypeAction(
N->getOperand(0).getValueType())
952 SDValue Res = GetPromotedInteger(
N->getOperand(0));
961 DAG.getValueType(
N->getOperand(0).getValueType()));
963 return DAG.getZeroExtendInReg(Res, dl,
N->getOperand(0).getValueType());
970 return DAG.getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
975 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
979 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT,
N->getChain(),
N->getBasePtr(),
980 N->getMemoryVT(),
N->getMemOperand());
989 assert(!
N->isIndexed() &&
"Indexed vp_load during type legalization!");
990 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
993 :
N->getExtensionType();
996 DAG.getExtLoadVP(ExtType, dl, NVT,
N->getChain(),
N->getBasePtr(),
997 N->getMask(),
N->getVectorLength(),
N->getMemoryVT(),
998 N->getMemOperand(),
N->isExpandingLoad());
1006 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1007 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
1014 SDValue Res = DAG.getMaskedLoad(NVT, dl,
N->getChain(),
N->getBasePtr(),
1015 N->getOffset(),
N->getMask(), ExtPassThru,
1016 N->getMemoryVT(),
N->getMemOperand(),
1017 N->getAddressingMode(), ExtType,
1018 N->isExpandingLoad());
1026 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1027 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
1029 "Gather result type and the passThru argument type should be the same");
1036 SDValue Ops[] = {
N->getChain(), ExtPassThru,
N->getMask(),
N->getBasePtr(),
1037 N->getIndex(),
N->getScale() };
1038 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
1039 N->getMemoryVT(), dl,
Ops,
1040 N->getMemOperand(),
N->getIndexType(),
1048SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_COMPRESS(
SDNode *
N) {
1049 SDValue Vec = GetPromotedInteger(
N->getOperand(0));
1050 SDValue Passthru = GetPromotedInteger(
N->getOperand(2));
1052 N->getOperand(1), Passthru);
1055SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE_VAR(
SDNode *
N) {
1056 SDValue V = GetPromotedInteger(
N->getOperand(0));
1065 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(1));
1066 EVT VT =
N->getValueType(0);
1067 EVT SVT = getSetCCResultType(VT);
1068 SDValue Ops[3] = {
N->getOperand(0),
N->getOperand(1) };
1069 unsigned NumOps =
N->getNumOperands();
1072 Ops[2] = PromoteTargetBoolean(
N->getOperand(2), VT);
1080 ReplaceValueWith(
SDValue(
N, 0), Res);
1083 return DAG.getBoolExtOrTrunc(Res.
getValue(1), dl, NVT, VT);
1098 unsigned Opcode =
N->getOpcode();
1104 SExtOrZExtPromotedOperands(Op1, Op2);
1110 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
1112 if (TLI.isSExtCheaperThanZExt(OVT, NVT)) {
1113 Op1 = SExtPromotedInteger(Op1);
1114 Op2 = SExtPromotedInteger(Op2);
1118 Op1 = ZExtPromotedInteger(Op1);
1119 Op2 = ZExtPromotedInteger(Op2);
1122 SDValue SatMax = DAG.getConstant(MaxVal, dl, NVT);
1131 Op1 = GetPromotedInteger(Op1);
1133 Op2 = ZExtPromotedInteger(Op2);
1135 Op1 = SExtPromotedInteger(Op1);
1136 Op2 = SExtPromotedInteger(Op2);
1143 if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) {
1156 "addition, subtraction or left shift");
1159 unsigned SHLAmount = NewBits - OldBits;
1161 DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl);
1166 SDValue Result = DAG.getNode(Opcode, dl, PromotedType, Op1, Op2);
1167 return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1173 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
1174 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
1175 SDValue Result = DAG.getNode(AddOp, dl, PromotedType, Op1, Op2);
1184 SDValue Op1Promoted, Op2Promoted;
1190 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1191 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1193 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1194 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1196 EVT OldType =
N->getOperand(0).getValueType();
1208 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
1209 SDValue Result = DAG.getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted,
1210 Op2Promoted,
N->getOperand(2));
1212 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
1213 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
1215 return DAG.getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1220 unsigned SatW,
bool Signed,
1223 EVT VT =
V.getValueType();
1250 EVT VT =
LHS.getValueType();
1266 assert(Res &&
"Expanding DIVFIX with wide type failed?");
1272 "Tried to saturate to more than the original type?");
1281 SDValue Op1Promoted, Op2Promoted;
1287 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1288 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1290 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1291 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1294 unsigned Scale =
N->getConstantOperandVal(2);
1298 if (TLI.isTypeLegal(PromotedType)) {
1300 TLI.getFixedPointOperationAction(
N->getOpcode(), PromotedType, Scale);
1303 N->getValueType(0).getScalarSizeInBits();
1307 DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1308 SDValue Res = DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted,
1309 Op2Promoted,
N->getOperand(2));
1312 DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1318 if (
SDValue Res = TLI.expandFixedPointDiv(
N->getOpcode(), dl, Op1Promoted,
1319 Op2Promoted, Scale, DAG)) {
1322 N->getValueType(0).getScalarSizeInBits(),
1330 N->getValueType(0).getScalarSizeInBits());
1333SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(
SDNode *
N,
unsigned ResNo) {
1335 return PromoteIntRes_Overflow(
N);
1339 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1340 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1341 EVT OVT =
N->getOperand(0).getValueType();
1342 EVT NVT =
LHS.getValueType();
1352 DAG.getValueType(OVT));
1354 Ofl = DAG.getSetCC(dl,
N->getValueType(1), Ofl, Res,
ISD::SETNE);
1357 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1363 EVT PromotedResultTy =
1364 TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1365 return DAG.
getNode(
N->getOpcode(), SDLoc(
N), PromotedResultTy,
1366 N->getOperand(0),
N->getOperand(1));
1372 SDValue LHS = GetPromotedInteger(
N->getOperand(1));
1373 SDValue RHS = GetPromotedInteger(
N->getOperand(2));
1375 unsigned Opcode =
N->getOpcode();
1376 if (Opcode == ISD::VP_MERGE)
1377 return DAG.getNode(Opcode, SDLoc(
N),
LHS.getValueType(), Mask,
LHS,
RHS,
1379 return DAG.getNode(Opcode, SDLoc(
N),
LHS.getValueType(), Mask,
LHS,
RHS);
1383 SDValue LHS = GetPromotedInteger(
N->getOperand(2));
1384 SDValue RHS = GetPromotedInteger(
N->getOperand(3));
1386 LHS.getValueType(),
N->getOperand(0),
1387 N->getOperand(1),
LHS,
RHS,
N->getOperand(4));
1392 EVT InVT =
N->getOperand(OpNo).getValueType();
1393 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1395 EVT SVT = getSetCCResultType(InVT);
1402 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1403 SVT = getSetCCResultType(InVT);
1411 assert(SVT.
isVector() ==
N->getOperand(OpNo).getValueType().isVector() &&
1412 "Vector compare must return a vector result!");
1416 if (
N->isStrictFPOpcode()) {
1417 SDVTList VTs = DAG.getVTList({SVT, MVT::Other});
1418 SDValue Opers[] = {
N->getOperand(0),
N->getOperand(1),
1419 N->getOperand(2),
N->getOperand(3)};
1420 SetCC = DAG.
getNode(
N->getOpcode(), dl, VTs, Opers,
N->getFlags());
1425 SetCC = DAG.
getNode(
N->getOpcode(), dl, SVT,
N->getOperand(0),
1426 N->getOperand(1),
N->getOperand(2),
N->getFlags());
1429 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1436 EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1441 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(1));
1442 EVT VT =
N->getValueType(0);
1446 DAG.
getNode(
N->getOpcode(), dl, DAG.getVTList(VT, NVT),
N->getOperand(0));
1448 ReplaceValueWith(
SDValue(
N, 0), Res);
1453 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1456 RHS = ZExtPromotedInteger(
RHS);
1457 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
LHS.getValueType(),
LHS,
RHS);
1460SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(
SDNode *
N) {
1461 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1463 Op.getValueType(),
Op,
N->getOperand(1));
1466SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(
SDNode *
N) {
1470 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1471 SDValue RHS = GetPromotedInteger(
N->getOperand(1));
1472 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
LHS.getValueType(),
LHS,
RHS);
1477 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1478 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1479 if (
N->getNumOperands() == 2)
1480 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
LHS.getValueType(),
LHS,
RHS);
1481 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1482 assert((
N->getOpcode() == ISD::VP_SDIV ||
N->getOpcode() == ISD::VP_SREM) &&
1483 "Expected VP opcode");
1486 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
LHS.getValueType(),
LHS,
RHS,
1492 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1493 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1494 if (
N->getNumOperands() == 2)
1495 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
LHS.getValueType(),
LHS,
RHS);
1496 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1497 assert((
N->getOpcode() == ISD::VP_UDIV ||
N->getOpcode() == ISD::VP_UREM) &&
1498 "Expected VP opcode");
1506SDValue DAGTypeLegalizer::PromoteIntRes_ZExtMaskedIntBinOp(
SDNode *
N) {
1507 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1508 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1510 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
LHS.getValueType(),
LHS,
RHS,
1514SDValue DAGTypeLegalizer::PromoteIntRes_SExtMaskedIntBinOp(
SDNode *
N) {
1515 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1516 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1518 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
LHS.getValueType(),
LHS,
RHS,
1522SDValue DAGTypeLegalizer::PromoteIntRes_SExtOrZExtIntBinOp(
SDNode *
N) {
1528 SExtOrZExtPromotedOperands(
LHS,
RHS);
1530 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
1536 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1539 RHS = ZExtPromotedInteger(
RHS);
1540 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
LHS.getValueType(),
LHS,
RHS);
1546 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1548 RHS = ZExtPromotedInteger(
RHS);
1549 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
LHS.getValueType(),
LHS,
RHS);
1554 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
1556 unsigned Opcode =
N->getOpcode();
1567 !TLI.isOperationLegalOrCustom(Opcode, VT) &&
1570 SDValue Op0 = GetPromotedInteger(
N->getOperand(0));
1572 Amt = ZExtPromotedInteger(Amt);
1578 DAG.getConstant(OldBits,
DL, AmtVT));
1579 SDValue HiShift = DAG.getShiftAmountConstant(OldBits, VT,
DL);
1581 SDValue Lo = DAG.getZeroExtendInReg(Op0,
DL, OldVT);
1593 SDValue Res = TLI.expandROT(
N,
true , DAG);
1594 ReplaceValueWith(
SDValue(
N, 0), Res);
1599 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1600 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1603 Amt = ZExtPromotedInteger(Amt);
1607 EVT OldVT =
N->getOperand(0).getValueType();
1608 EVT VT =
Lo.getValueType();
1609 unsigned Opcode =
N->getOpcode();
1616 DAG.getConstant(OldBits,
DL, AmtVT));
1624 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1625 SDValue HiShift = DAG.getShiftAmountConstant(OldBits, VT,
DL);
1627 Lo = DAG.getZeroExtendInReg(
Lo,
DL, OldVT);
1637 DAG.getShiftAmountConstant(NewBits - OldBits, VT,
DL));
1643 DAG.getConstant(NewBits - OldBits,
DL, AmtVT));
1645 return DAG.getNode(Opcode,
DL, VT,
Hi,
Lo, Amt);
1649 unsigned Opcode =
N->getOpcode();
1652 EVT OldVT =
N->getOperand(0).getValueType();
1653 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
1658 if (!TLI.isOperationLegalOrCustomOrPromote(
ISD::CLMUL, VT) &&
1660 TLI.isOperationLegalOrCustom(
1661 ISD::CLMUL, TLI.getRegisterType(*DAG.getContext(), VT)))) {
1662 if (
SDValue Res = TLI.expandCLMUL(
N, DAG))
1665 SDValue X = GetPromotedInteger(
N->getOperand(0));
1666 SDValue Y = GetPromotedInteger(
N->getOperand(1));
1670 SDValue X = ZExtPromotedInteger(
N->getOperand(0));
1671 SDValue Y = ZExtPromotedInteger(
N->getOperand(1));
1675 if (NewBits < 2 * OldBits) {
1677 unsigned ShAmt = Opcode ==
ISD::CLMULH ? OldBits : OldBits - 1;
1679 DAG.getShiftAmountConstant(ShAmt, VT,
DL));
1681 ShAmt = Opcode ==
ISD::CLMULH ? NewBits - OldBits : NewBits - OldBits + 1;
1683 DAG.getShiftAmountConstant(ShAmt, VT,
DL));
1688 unsigned ShAmt = Opcode ==
ISD::CLMULH ? OldBits : OldBits - 1;
1690 DAG.getShiftAmountConstant(ShAmt, VT,
DL));
1695 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1696 if (!TLI.isOperationLegalOrCustomOrPromote(
ISD::PEXT, VT)) {
1697 if (
SDValue Res = TLI.expandPEXT(
N, DAG))
1702 SDValue X = GetPromotedInteger(
N->getOperand(0));
1703 SDValue Y = ZExtPromotedInteger(
N->getOperand(1));
1709 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1710 if (!TLI.isOperationLegalOrCustomOrPromote(
ISD::PDEP, VT)) {
1711 if (
SDValue Res = TLI.expandPDEP(
N, DAG))
1716 SDValue X = GetPromotedInteger(
N->getOperand(0));
1717 SDValue Y = GetPromotedInteger(
N->getOperand(1));
1723 EVT VT =
N->getValueType(0);
1724 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1731 LHS = SExtPromotedInteger(
N->getOperand(0));
1732 RHS = SExtPromotedInteger(
N->getOperand(1));
1734 LHS = ZExtPromotedInteger(
N->getOperand(0));
1735 RHS = ZExtPromotedInteger(
N->getOperand(1));
1742 DAG.getShiftAmountConstant(BW, NVT, dl));
1746 DAG.getShiftAmountConstant(NBW - BW, NVT, dl));
1747 return DAG.getNode(
N->getOpcode(), dl, NVT,
LHS,
RHS);
1751 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1763 Res = GetPromotedInteger(InOp);
1770 "Dst and Src must have the same number of elements");
1772 "Promoted vector type must be a power of two");
1775 GetSplitVector(InOp, EOp1, EOp2);
1784 SDValue WideInOp = GetWidenedVector(InOp);
1789 N->getValueType(0).getScalarType(), NumElem);
1798 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1807SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(
SDNode *
N,
unsigned ResNo) {
1809 return PromoteIntRes_Overflow(
N);
1813 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1814 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1815 EVT OVT =
N->getOperand(0).getValueType();
1816 EVT NVT =
LHS.getValueType();
1825 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1827 Ofl = DAG.getSetCC(dl,
N->getValueType(1), Ofl, Res,
ISD::SETNE);
1830 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1841 return PromoteIntRes_Overflow(
N);
1853 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1854 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1856 EVT ValueVTs[] = {
LHS.getValueType(),
N->getValueType(1)};
1859 SDValue Res = DAG.
getNode(
N->getOpcode(), SDLoc(
N), DAG.getVTList(ValueVTs),
1870 assert(ResNo == 1 &&
"Don't know how to promote other results yet.");
1871 return PromoteIntRes_Overflow(
N);
1875 EVT OVT =
N->getValueType(0);
1876 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
1882 !TLI.isOperationLegalOrCustomOrPromote(
ISD::ABS, NVT) &&
1884 !TLI.isOperationLegal(
ISD::SMAX, NVT)) {
1885 if (
SDValue Res = TLI.expandABS(
N, DAG))
1889 SDValue Op0 = SExtPromotedInteger(
N->getOperand(0));
1893SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(
SDNode *
N,
unsigned ResNo) {
1896 return PromoteIntRes_Overflow(
N);
1900 EVT SmallVT =
LHS.getValueType();
1907 LHS = SExtPromotedInteger(
LHS);
1908 RHS = SExtPromotedInteger(
RHS);
1910 LHS = ZExtPromotedInteger(
LHS);
1911 RHS = ZExtPromotedInteger(
RHS);
1913 SDVTList VTs = DAG.getVTList(
LHS.getValueType(),
N->getValueType(1));
1925 DAG.getShiftAmountConstant(Shift,
Mul.getValueType(),
DL));
1926 Overflow = DAG.getSetCC(
DL,
N->getValueType(1),
Hi,
1927 DAG.getConstant(0,
DL,
Hi.getValueType()),
1932 Mul, DAG.getValueType(SmallVT));
1942 ReplaceValueWith(
SDValue(
N, 1), Overflow);
1947 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1948 N->getValueType(0)));
1952 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1954 const APInt &MulImm =
N->getConstantOperandAPInt(0);
1961 EVT VT =
N->getValueType(0);
1964 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1965 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1969 for (
unsigned i = 0; i < NumRegs; ++i) {
1970 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr,
N->getOperand(2),
1971 N->getConstantOperandVal(3));
1976 if (DAG.getDataLayout().isBigEndian())
1977 std::reverse(Parts.begin(), Parts.end());
1980 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
1982 for (
unsigned i = 1; i < NumRegs; ++i) {
1987 DAG.getShiftAmountConstant(i * RegVT.
getSizeInBits(), NVT, dl));
1993 ReplaceValueWith(
SDValue(
N, 1), Chain);
2006bool DAGTypeLegalizer::PromoteIntegerOperand(
SDNode *
N,
unsigned OpNo) {
2009 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
2014 switch (
N->getOpcode()) {
2017 dbgs() <<
"PromoteIntegerOperand Op #" << OpNo <<
": ";
2018 N->dump(&DAG);
dbgs() <<
"\n";
2024 Res = PromoteIntOp_ANY_EXTEND_VECTOR_INREG(
N);
2030 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(
N, OpNo);
break;
2031 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(
N, OpNo);
break;
2036 Res = PromoteIntOp_COND_LOOP(
N, OpNo);
2040 Res = PromoteIntOp_FAKE_USE(
N);
2043 Res = PromoteIntOp_INSERT_VECTOR_ELT(
N, OpNo);
2047 Res = PromoteIntOp_ScalarOp(
N);
2050 case ISD::SELECT: Res = PromoteIntOp_SELECT(
N, OpNo);
break;
2052 case ISD::SETCC: Res = PromoteIntOp_SETCC(
N, OpNo);
break;
2067 case ISD::VP_GATHER:
2072 case ISD::VP_SCATTER:
2076 Res = PromoteIntOp_VECTOR_COMPRESS(
N, OpNo);
2079 Res = PromoteIntOp_VECTOR_SHUFFLE_VAR(
N, OpNo);
2086 Res = PromoteIntOp_CONVERT_FROM_ARBITRARY_FP(
N);
2101 Res = PromoteIntOp_Shift(
N);
2105 case ISD::UCMP: Res = PromoteIntOp_CMP(
N);
break;
2108 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(
N);
break;
2134 case ISD::VP_REDUCE_ADD:
2135 case ISD::VP_REDUCE_MUL:
2136 case ISD::VP_REDUCE_AND:
2137 case ISD::VP_REDUCE_OR:
2138 case ISD::VP_REDUCE_XOR:
2139 case ISD::VP_REDUCE_SMAX:
2140 case ISD::VP_REDUCE_SMIN:
2141 case ISD::VP_REDUCE_UMAX:
2142 case ISD::VP_REDUCE_UMIN:
2143 Res = PromoteIntOp_VP_REDUCE(
N, OpNo);
2148 Res = PromoteIntOp_STACKMAP(
N, OpNo);
2151 Res = PromoteIntOp_PATCHPOINT(
N, OpNo);
2154 Res = PromoteIntOp_WRITE_REGISTER(
N, OpNo);
2156 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
2157 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2158 Res = PromoteIntOp_VP_STRIDED(
N, OpNo);
2160 case ISD::EXPERIMENTAL_VP_SPLICE:
2161 Res = PromoteIntOp_VP_SPLICE(
N, OpNo);
2164 Res = PromoteIntOp_VECTOR_HISTOGRAM(
N, OpNo);
2169 Res = PromoteIntOp_UnaryBooleanVectorOp(
N, OpNo);
2172 Res = PromoteIntOp_GET_ACTIVE_LANE_MASK(
N);
2175 Res = PromoteIntOp_VECTOR_MATCH(
N, OpNo);
2181 Res = PromoteIntOp_MaskedBinOp(
N, OpNo);
2186 Res = PromoteIntOp_PARTIAL_REDUCE_MLA(
N);
2189 Res = PromoteIntOp_VECTOR_REPEAT(
N);
2193 Res = PromoteIntOp_LOOP_DEPENDENCE_MASK(
N);
2198 if (!Res.
getNode())
return false;
2205 const bool IsStrictFp =
N->isStrictFPOpcode();
2207 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
2208 "Invalid operand expansion");
2212 ReplaceValueWith(
SDValue(
N, 0), Res);
2226 if (TLI.isSExtCheaperThanZExt(
LHS.getValueType(), OpL.
getValueType())) {
2230 unsigned OpLEffectiveBits =
2231 DAG.computeKnownBits(OpL).countMaxActiveBits();
2232 unsigned OpREffectiveBits =
2233 DAG.computeKnownBits(OpR).countMaxActiveBits();
2234 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
2235 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
2242 LHS = SExtPromotedInteger(
LHS);
2243 RHS = SExtPromotedInteger(
RHS);
2252 unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL);
2253 unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR);
2254 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
2255 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
2262 LHS = ZExtPromotedInteger(
LHS);
2263 RHS = ZExtPromotedInteger(
RHS);
2277 LHS = SExtPromotedInteger(
LHS);
2278 RHS = SExtPromotedInteger(
RHS);
2283 "Unknown integer comparison!");
2285 SExtOrZExtPromotedOperands(
LHS,
RHS);
2289 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2293SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND_VECTOR_INREG(
SDNode *
N) {
2294 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2295 EVT ResVT =
N->getValueType(0);
2296 EVT OpVT =
Op.getValueType();
2299 Op = DAG.getExtractSubvector(SDLoc(
Op), NewVT,
Op, 0);
2304 SDValue Op1 = GetPromotedInteger(
N->getOperand(1));
2305 return DAG.getAtomic(
N->getOpcode(), SDLoc(
N),
N->getMemoryVT(),
2306 N->getChain(), Op1,
N->getBasePtr(),
N->getMemOperand());
2310 EVT OutVT =
N->getValueType(0);
2313 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
2316 switch (getTypeAction(InVT)) {
2322 if (OutEltBits % InEltBits == 0 && isTypeLegal(IntVT)) {
2325 SDValue Promoted = GetPromotedInteger(InOp);
2328 if (PromotedEltBits % InEltBits == 0) {
2329 unsigned Scale = PromotedEltBits / InEltBits;
2337 PromotedBits / OutEltBits);
2338 if (isTypeLegal(PackedVT) && isTypeLegal(CastVT) &&
2345 Mask[
I] =
I * Scale +
2346 (DAG.getDataLayout().isBigEndian() ? Scale - 1 : 0);
2347 if (TLI.isShuffleMaskLegal(Mask, PackedVT)) {
2350 Packed = DAG.getVectorShuffle(PackedVT, dl, Packed,
2351 DAG.getUNDEF(PackedVT), Mask);
2354 return DAG.getExtractSubvector(dl, OutVT, Packed, 0);
2355 return DAG.getExtractVectorElt(dl, OutVT, Packed, 0);
2360 unsigned PartsPerElt = OutEltBits / InEltBits;
2364 for (
unsigned I = 0;
I != NumElts; ++
I) {
2366 for (
unsigned J = 0; J != PartsPerElt; ++J) {
2367 SDValue Part = DAG.getExtractVectorElt(
2369 I * PartsPerElt + J);
2370 Part = DAG.getZeroExtendInReg(Part, dl, InVT.
getScalarType());
2371 Part = DAG.getZExtOrTrunc(Part, dl, IntVT);
2372 unsigned Shift = DAG.getDataLayout().isBigEndian()
2373 ? (PartsPerElt - 1 - J) * InEltBits
2376 DAG.getShiftAmountConstant(Shift, IntVT, dl));
2384 Packed = DAG.getBuildVector(VecVT, dl, Elts);
2391 DAG.getDataLayout().isLittleEndian()) {
2401 if (isTypeLegal(WideVecVT)) {
2402 SDValue Promoted = GetPromotedInteger(InOp);
2405 DAG.getVectorIdxConstant(0, dl));
2418 return CreateStackStoreLoad(InOp, OutVT);
2421SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(
SDNode *
N,
unsigned OpNo) {
2422 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
2426 PromoteSetCCOperands(
LHS,
RHS,
N->getOperandCondCode(1));
2430 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
2431 N->getOperand(1),
LHS,
RHS,
N->getOperand(4)),
2435SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(
SDNode *
N,
unsigned OpNo) {
2436 assert(OpNo == 1 &&
"only know how to promote condition");
2439 SDValue Cond = PromoteTargetBoolean(
N->getOperand(1), MVT::Other);
2442 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
Cond,
2443 N->getOperand(2)), 0);
2446SDValue DAGTypeLegalizer::PromoteIntOp_COND_LOOP(
SDNode *
N,
unsigned OpNo) {
2447 assert(OpNo == 1 &&
"only know how to promote condition");
2450 SDValue Cond = PromoteTargetBoolean(
N->getOperand(1), MVT::Other);
2453 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
Cond), 0);
2458 EVT OVT =
N->getOperand(0).getValueType();
2459 SDValue Lo = ZExtPromotedInteger(
N->getOperand(0));
2460 SDValue Hi = GetPromotedInteger(
N->getOperand(1));
2461 assert(
Lo.getValueType() ==
N->getValueType(0) &&
"Operand over promoted?");
2466 DAG.getShiftAmountConstant(OVT.
getSizeInBits(),
N->getValueType(0), dl));
2467 return DAG.getNode(
ISD::OR, dl,
N->getValueType(0),
Lo,
Hi);
2476 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
2477 "Legal vector of one illegal element?");
2482 assert(
N->getOperand(0).getValueSizeInBits() >=
2483 N->getValueType(0).getScalarSizeInBits() &&
2484 "Type of inserted value narrower than vector element type!");
2487 for (
unsigned i = 0; i < NumElts; ++i)
2488 NewOps.
push_back(GetPromotedInteger(
N->getOperand(i)));
2490 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
2493SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(
SDNode *
N,
2500 assert(
N->getOperand(1).getValueSizeInBits() >=
2501 N->getValueType(0).getScalarSizeInBits() &&
2502 "Type of inserted value narrower than vector element type!");
2503 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
2504 GetPromotedInteger(
N->getOperand(1)),
2509 assert(OpNo == 2 &&
"Different operand and result vector types?");
2512 SDValue Idx = DAG.getZExtOrTrunc(
N->getOperand(2), SDLoc(
N),
2513 TLI.getVectorIdxTy(DAG.getDataLayout()));
2514 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
2515 N->getOperand(1), Idx), 0);
2519 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2523 return SDValue(DAG.UpdateNodeOperands(
N,
Op), 0);
2526SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(
SDNode *
N,
unsigned OpNo) {
2527 assert(OpNo == 0 &&
"Only know how to promote the condition!");
2529 EVT OpTy =
N->getOperand(1).getValueType();
2532 if (
SDValue Res = WidenVSELECTMask(
N))
2533 return DAG.getNode(
N->getOpcode(), SDLoc(
N),
N->getValueType(0),
2534 Res,
N->getOperand(1),
N->getOperand(2));
2538 Cond = PromoteTargetBoolean(
Cond, OpVT);
2540 return SDValue(DAG.UpdateNodeOperands(
N,
Cond,
N->getOperand(1),
2541 N->getOperand(2)), 0);
2544SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2545 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2549 PromoteSetCCOperands(
LHS,
RHS,
N->getOperandCondCode(4));
2553 N->getOperand(3),
N->getOperand(4)), 0);
2556SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2557 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2561 PromoteSetCCOperands(
LHS,
RHS,
N->getOperandCondCode(2));
2564 return SDValue(DAG.UpdateNodeOperands(
N,
LHS,
RHS,
N->getOperand(2)), 0);
2568 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
2569 ZExtPromotedInteger(
N->getOperand(1))), 0);
2577 LHS = SExtPromotedInteger(
LHS);
2578 RHS = SExtPromotedInteger(
RHS);
2580 SExtOrZExtPromotedOperands(
LHS,
RHS);
2587 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
N->getOperand(1),
2588 ZExtPromotedInteger(
N->getOperand(2))), 0);
2592 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2596 Op, DAG.getValueType(
N->getOperand(0).getValueType()));
2600 return SDValue(DAG.UpdateNodeOperands(
N,
2601 SExtPromotedInteger(
N->getOperand(0))), 0);
2604SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(
SDNode *
N) {
2605 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
2606 SExtPromotedInteger(
N->getOperand(1))), 0);
2611 SDValue Ch =
N->getChain(), Ptr =
N->getBasePtr();
2614 SDValue Val = GetPromotedInteger(
N->getValue());
2617 return DAG.getTruncStore(Ch, dl, Val, Ptr,
2618 N->getMemoryVT(),
N->getMemOperand());
2624 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2625 assert(!
N->isIndexed() &&
"expecting unindexed vp_store!");
2627 SDValue DataOp = GetPromotedInteger(
N->getValue());
2628 return DAG.getTruncStoreVP(
N->getChain(), SDLoc(
N), DataOp,
N->getBasePtr(),
2629 N->getMask(),
N->getVectorLength(),
2630 N->getMemoryVT(),
N->getMemOperand(),
2631 N->isCompressingStore());
2642 Mask = PromoteTargetBoolean(Mask, DataVT);
2645 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
2648 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2649 DataOp = GetPromotedInteger(DataOp);
2651 return DAG.getMaskedStore(
N->getChain(), SDLoc(
N), DataOp,
N->getBasePtr(),
2652 N->getOffset(), Mask,
N->getMemoryVT(),
2653 N->getMemOperand(),
N->getAddressingMode(),
2654 true,
N->isCompressingStore());
2659 assert(OpNo == 3 &&
"Only know how to promote the mask!");
2660 EVT DataVT =
N->getValueType(0);
2661 SDValue Mask = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2663 NewOps[OpNo] =
Mask;
2664 SDNode *Res = DAG.UpdateNodeOperands(
N, NewOps);
2680 EVT DataVT =
N->getValueType(0);
2681 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2682 }
else if (OpNo == 4) {
2684 if (
N->isIndexSigned())
2686 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2688 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2690 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2692 SDNode *Res = DAG.UpdateNodeOperands(
N, NewOps);
2704 assert(OpNo == 2 &&
"Only the index operand can be promoted");
2707 NewOps[OpNo] =
N->isIndexSigned() ? SExtPromotedInteger(
N->getOperand(OpNo))
2708 : ZExtPromotedInteger(
N->getOperand(OpNo));
2709 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
2714 bool TruncateStore =
N->isTruncatingStore();
2719 EVT DataVT =
N->getValue().getValueType();
2720 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2721 }
else if (OpNo == 4) {
2723 if (
N->isIndexSigned())
2725 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2727 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2729 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2730 TruncateStore =
true;
2733 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
N->getMemoryVT(),
2734 SDLoc(
N), NewOps,
N->getMemOperand(),
2735 N->getIndexType(), TruncateStore);
2740 assert(OpNo == 3 &&
"Only the index operand can be promoted");
2743 NewOps[OpNo] =
N->isIndexSigned() ? SExtPromotedInteger(
N->getOperand(OpNo))
2744 : ZExtPromotedInteger(
N->getOperand(OpNo));
2745 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
2750 assert(OpNo == 1 &&
"Can only promote VECTOR_COMPRESS mask.");
2754 SDValue Mask = PromoteTargetBoolean(
N->getOperand(1), VT);
2758SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_SHUFFLE_VAR(
SDNode *
N,
2760 assert(OpNo == 1 &&
"The source shares the result's type, so promoting it "
2761 "is a result promotion.");
2764 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0), Mask), 0);
2768 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2773 return SDValue(DAG.UpdateNodeOperands(
N,
2774 ZExtPromotedInteger(
N->getOperand(0))), 0);
2777SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_FROM_ARBITRARY_FP(
SDNode *
N) {
2778 return SDValue(DAG.UpdateNodeOperands(
N, GetPromotedInteger(
N->getOperand(0)),
2783SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(
SDNode *
N) {
2784 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
2785 ZExtPromotedInteger(
N->getOperand(1))), 0);
2792 EVT VT =
N->getValueType(0);
2797 if (
N->getFlags().hasNonNeg() &&
Op.getValueType() == VT &&
2798 TLI.isSExtCheaperThanZExt(Src.getValueType(), VT)) {
2799 unsigned OpEffectiveBits = DAG.ComputeMaxSignificantBits(
Op);
2800 if (OpEffectiveBits <= Src.getScalarValueSizeInBits())
2805 return DAG.getZeroExtendInReg(
Op, dl, Src.getValueType());
2809 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2811 DAG.UpdateNodeOperands(
N,
N->getOperand(0),
N->getOperand(1), Op2), 0);
2814SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(
SDNode *
N) {
2816 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
2817 return SDValue(DAG.UpdateNodeOperands(
N,
Op), 0);
2826 unsigned OpOffset = IsStrict ? 1 : 0;
2830 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(
N->getValueType(0))
2833 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
2834 if (LCImpl == RTLIB::Unsupported) {
2840 if (IsPowI &&
N->getValueType(0).isVector())
2841 return DAG.UnrollVectorOp(
N);
2843 NewOps[1 + OpOffset] = SExtPromotedInteger(
N->getOperand(1 + OpOffset));
2844 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
2856 if (
N->getOperand(1 + OpOffset).getScalarValueSizeInBits() >
2857 DAG.getLibInfo().getIntSize()) {
2858 const Function &Fn = DAG.getMachineFunction().getFunction();
2859 Fn.getContext().diagnose(DiagnosticInfoLegalizationFailure(
2860 Twine(IsPowI ?
"powi" :
"ldexp") +
2861 " exponent does not match sizeof(int)",
2862 Fn, N->getDebugLoc()));
2864 ReplaceValueWith(SDValue(N, 1), Chain);
2865 ReplaceValueWith(SDValue(N, 0), DAG.getPOISON(N->getValueType(0)));
2869 TargetLowering::MakeLibCallOptions CallOptions;
2870 CallOptions.setIsSigned(
true);
2871 SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)};
2872 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
2873 DAG, LCImpl,
N->getValueType(0),
Ops, CallOptions, SDLoc(
N), Chain);
2874 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2876 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2881 switch (
N->getOpcode()) {
2889 case ISD::VP_REDUCE_ADD:
2890 case ISD::VP_REDUCE_MUL:
2891 case ISD::VP_REDUCE_AND:
2892 case ISD::VP_REDUCE_OR:
2893 case ISD::VP_REDUCE_XOR:
2897 case ISD::VP_REDUCE_SMAX:
2898 case ISD::VP_REDUCE_SMIN:
2902 case ISD::VP_REDUCE_UMAX:
2903 case ISD::VP_REDUCE_UMIN:
2913 return GetPromotedInteger(
V);
2915 return SExtPromotedInteger(
V);
2917 return ZExtPromotedInteger(
V);
2923 SDValue Op = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
2925 EVT OrigEltVT =
N->getOperand(0).getValueType().getVectorElementType();
2926 EVT InVT =
Op.getValueType();
2928 EVT ResVT =
N->getValueType(0);
2929 unsigned Opcode =
N->getOpcode();
2946 switch (TLI.getBooleanContents(InVT)) {
2949 Op = ZExtPromotedInteger(
N->getOperand(0));
2952 Op = SExtPromotedInteger(
N->getOperand(0));
2965 switch (TLI.getBooleanContents(InVT)) {
2968 Op = ZExtPromotedInteger(
N->getOperand(0));
2971 Op = SExtPromotedInteger(
N->getOperand(0));
2977 return DAG.getNode(Opcode, SDLoc(
N), ResVT,
Op);
2981 SDValue Reduce = DAG.getNode(Opcode, dl, EltVT,
Op);
2985SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
2992 NewOps[2] = PromoteTargetBoolean(
Op,
N->getOperand(1).getValueType());
2993 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
2996 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2998 Op = PromoteIntOpVectorReduction(
N,
Op);
3002 EVT VT =
N->getValueType(0);
3003 EVT EltVT =
Op.getValueType().getScalarType();
3006 return DAG.getNode(
N->getOpcode(), SDLoc(
N), VT, NewOps);
3018 SDValue Op = ZExtPromotedInteger(
N->getOperand(1));
3019 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
Op), 0);
3022SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
3025 NewOps[OpNo] = GetPromotedInteger(NewOps[OpNo]);
3026 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
3029SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
3032 NewOps[OpNo] = GetPromotedInteger(NewOps[OpNo]);
3033 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
3038 const Function &Fn = DAG.getMachineFunction().getFunction();
3040 "cannot use llvm.write_register with illegal type", Fn,
3042 return N->getOperand(0);
3045SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
3046 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
3047 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
3050 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
3051 SDNode *Res = DAG.UpdateNodeOperands(
N, NewOps);
3061SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(
SDNode *
N,
unsigned OpNo) {
3065 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
3066 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
3069 assert((OpNo == 4 || OpNo == 5) &&
"Unexpected operand for promotion");
3071 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
3072 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
3075SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_HISTOGRAM(
SDNode *
N,
3077 assert(OpNo == 1 &&
"Unexpected operand for promotion");
3079 NewOps[1] = GetPromotedInteger(
N->getOperand(1));
3080 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
3083SDValue DAGTypeLegalizer::PromoteIntOp_UnaryBooleanVectorOp(
SDNode *
N,
3085 assert(OpNo == 0 &&
"Unexpected operand for promotion");
3089 if (TLI.getBooleanContents(
Op.getValueType()) ==
3091 NewOp = SExtPromotedInteger(
Op);
3093 NewOp = ZExtPromotedInteger(
Op);
3095 return SDValue(DAG.UpdateNodeOperands(
N, NewOp), 0);
3098SDValue DAGTypeLegalizer::PromoteIntOp_GET_ACTIVE_LANE_MASK(
SDNode *
N) {
3100 NewOps[0] = ZExtPromotedInteger(
N->getOperand(0));
3101 NewOps[1] = ZExtPromotedInteger(
N->getOperand(1));
3102 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
3105SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_MATCH(
SDNode *
N,
unsigned OpNo) {
3106 assert(OpNo < 3 &&
"Unexpected operand for promotion");
3108 return TLI.expandVectorMatch(
N, DAG);
3111 NewOps[2] = PromoteTargetBoolean(
N->getOperand(2),
N->getValueType(0));
3112 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
3115SDValue DAGTypeLegalizer::PromoteIntOp_MaskedBinOp(
SDNode *
N,
unsigned OpNo) {
3118 NewOps[2] = PromoteTargetBoolean(NewOps[2],
N->getValueType(0));
3119 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
3122SDValue DAGTypeLegalizer::PromoteIntOp_PARTIAL_REDUCE_MLA(
SDNode *
N) {
3124 switch (
N->getOpcode()) {
3126 NewOps[1] = SExtPromotedInteger(
N->getOperand(1));
3127 NewOps[2] = SExtPromotedInteger(
N->getOperand(2));
3130 NewOps[1] = ZExtPromotedInteger(
N->getOperand(1));
3131 NewOps[2] = ZExtPromotedInteger(
N->getOperand(2));
3134 NewOps[1] = SExtPromotedInteger(
N->getOperand(1));
3135 NewOps[2] = ZExtPromotedInteger(
N->getOperand(2));
3140 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
3143SDValue DAGTypeLegalizer::PromoteIntOp_LOOP_DEPENDENCE_MASK(
SDNode *
N) {
3145 NewOps[0] = ZExtPromotedInteger(
N->getOperand(0));
3146 NewOps[1] = ZExtPromotedInteger(
N->getOperand(1));
3147 NewOps[2] = ZExtPromotedInteger(
N->getOperand(2));
3149 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
3154 SDValue Src = GetPromotedInteger(
N->getOperand(0));
3155 EVT SrcVT = Src.getValueType();
3156 EVT OrigVT =
N->getValueType(0);
3171void DAGTypeLegalizer::ExpandIntegerResult(
SDNode *
N,
unsigned ResNo) {
3177 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
3180 switch (
N->getOpcode()) {
3183 dbgs() <<
"ExpandIntegerResult #" << ResNo <<
": ";
3184 N->dump(&DAG);
dbgs() <<
"\n";
3213 ExpandIntRes_ABS(
N,
Lo,
Hi);
3265 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(
N);
3266 SplitInteger(Tmp.first,
Lo,
Hi);
3267 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3272 SDVTList VTs = DAG.getVTList(
N->getValueType(0), MVT::Other);
3273 SDValue Tmp = DAG.getAtomicCmpSwap(
3275 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
3284 SplitInteger(Tmp,
Lo,
Hi);
3363 ExpandIntRes_Rotate(
N,
Lo,
Hi);
3368 ExpandIntRes_FunnelShift(
N,
Lo,
Hi);
3374 ExpandIntRes_CLMUL(
N,
Lo,
Hi);
3378 ExpandIntRes_PEXT(
N,
Lo,
Hi);
3382 ExpandIntRes_PDEP(
N,
Lo,
Hi);
3387 ExpandIntRes_MULH(
N,
Lo,
Hi);
3391 ExpandIntRes_VSCALE(
N,
Lo,
Hi);
3395 ExpandIntRes_READ_REGISTER(
N,
Lo,
Hi);
3400 ExpandIntRes_CTTZ_ELTS(
N,
Lo,
Hi);
3410std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(
SDNode *Node) {
3411 unsigned Opc =
Node->getOpcode();
3414 "expanded atomic requires a libcall action for the memory type");
3416 RTLIB::AtomicLibcall LC =
3418 EVT RetVT =
Node->getValueType(0);
3419 TargetLowering::MakeLibCallOptions CallOptions;
3423 Ops.push_back(
Node->getOperand(1));
3427 return TLI.makeLibCall(DAG, LC.
Impl, RetVT,
Ops, CallOptions, SDLoc(Node),
3428 Node->getOperand(0));
3433void DAGTypeLegalizer::ExpandShiftByConstant(
SDNode *
N,
const APInt &Amt,
3438 GetExpandedInteger(
N->getOperand(0), InL, InH);
3453 if (Amt.
uge(VTBits)) {
3454 Lo =
Hi = DAG.getConstant(0,
DL, NVT);
3455 }
else if (Amt.
ugt(NVTBits)) {
3456 Lo = DAG.getConstant(0,
DL, NVT);
3458 DAG.getShiftAmountConstant(Amt - NVTBits, NVT,
DL));
3459 }
else if (Amt == NVTBits) {
3460 Lo = DAG.getConstant(0,
DL, NVT);
3464 DAG.getShiftAmountConstant(Amt, NVT,
DL));
3466 if (TLI.isOperationLegal(
ISD::FSHL, NVT)) {
3468 DAG.getShiftAmountConstant(Amt, NVT,
DL));
3473 DAG.getShiftAmountConstant(Amt, NVT,
DL)),
3475 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT,
DL)));
3482 if (Amt.
uge(VTBits)) {
3483 Lo =
Hi = DAG.getConstant(0,
DL, NVT);
3484 }
else if (Amt.
ugt(NVTBits)) {
3486 DAG.getShiftAmountConstant(Amt - NVTBits, NVT,
DL));
3487 Hi = DAG.getConstant(0,
DL, NVT);
3488 }
else if (Amt == NVTBits) {
3490 Hi = DAG.getConstant(0,
DL, NVT);
3493 if (TLI.isOperationLegal(
ISD::FSHR, NVT)) {
3495 DAG.getShiftAmountConstant(Amt, NVT,
DL));
3500 DAG.getShiftAmountConstant(Amt, NVT,
DL)),
3502 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT,
DL)));
3505 DAG.getShiftAmountConstant(Amt, NVT,
DL));
3511 if (Amt.
uge(VTBits)) {
3513 DAG.getShiftAmountConstant(NVTBits - 1, NVT,
DL));
3514 }
else if (Amt.
ugt(NVTBits)) {
3516 DAG.getShiftAmountConstant(Amt - NVTBits, NVT,
DL));
3518 DAG.getShiftAmountConstant(NVTBits - 1, NVT,
DL));
3519 }
else if (Amt == NVTBits) {
3522 DAG.getShiftAmountConstant(NVTBits - 1, NVT,
DL));
3525 if (TLI.isOperationLegal(
ISD::FSHR, NVT)) {
3527 DAG.getShiftAmountConstant(Amt, NVT,
DL));
3532 DAG.getShiftAmountConstant(Amt, NVT,
DL)),
3534 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT,
DL)));
3537 DAG.getShiftAmountConstant(Amt, NVT,
DL));
3545bool DAGTypeLegalizer::
3547 unsigned Opc =
N->getOpcode();
3550 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
3555 "Expanded integer type size not a power of two!");
3559 KnownBits
Known = DAG.computeKnownBits(Amt);
3562 if (((
Known.Zero |
Known.One) & HighBitMask) == 0)
3567 GetExpandedInteger(In, InL, InH);
3571 if (
Known.One.intersects(HighBitMask)) {
3574 DAG.getConstant(~HighBitMask, dl, ShTy));
3579 Lo = DAG.getConstant(0, dl, NVT);
3580 Hi = DAG.getNode(
ISD::SHL, dl, NVT, InL, Amt);
3583 Hi = DAG.getConstant(0, dl, NVT);
3584 Lo = DAG.getNode(
ISD::SRL, dl, NVT, InH, Amt);
3588 DAG.getConstant(NVTBits - 1, dl, ShTy));
3589 Lo = DAG.getNode(
ISD::SRA, dl, NVT, InH, Amt);
3601 DAG.getConstant(NVTBits - 1, dl, ShTy));
3617 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
3619 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
3621 Lo = DAG.getNode(
Opc, dl, NVT, InL, Amt);
3622 Hi = DAG.getNode(
ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
3634bool DAGTypeLegalizer::
3637 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
3641 "Expanded integer type size not a power of two!");
3646 GetExpandedInteger(
N->getOperand(0), InL, InH);
3648 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
3651 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
3654 Amt, DAG.getConstant(0, dl, ShTy),
3658 switch (
N->getOpcode()) {
3664 DAG.getNode(
ISD::SHL, dl, NVT, InH, Amt),
3665 DAG.getNode(
ISD::SRL, dl, NVT, InL, AmtLack));
3668 LoL = DAG.getConstant(0, dl, NVT);
3671 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
3672 Hi = DAG.getSelect(dl, NVT,
isZero, InH,
3673 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
3679 DAG.getNode(
ISD::SRL, dl, NVT, InL, Amt),
3682 DAG.getNode(
ISD::SHL, dl, NVT, InH, AmtLack));
3685 HiL = DAG.getConstant(0, dl, NVT);
3688 Lo = DAG.getSelect(dl, NVT,
isZero, InL,
3689 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
3690 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
3696 DAG.getNode(
ISD::SRL, dl, NVT, InL, Amt),
3697 DAG.getNode(
ISD::SHL, dl, NVT, InH, AmtLack));
3701 DAG.getConstant(NVTBits - 1, dl, ShTy));
3704 Lo = DAG.getSelect(dl, NVT,
isZero, InL,
3705 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
3706 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
3731 EVT NewVT = getSetCCResultType(
LHS.getValueType());
3736 Res = DAG.getBoolExtOrTrunc(Res,
DL,
N->getValueType(0), NewVT);
3737 SplitInteger(Res,
Lo,
Hi);
3740void DAGTypeLegalizer::ExpandIntRes_MINMAX(
SDNode *
N,
3749 unsigned NumBits =
N->getValueType(0).getScalarSizeInBits();
3750 unsigned NumHalfBits = NumBits / 2;
3751 if (DAG.ComputeNumSignBits(
LHS) > NumHalfBits &&
3752 DAG.ComputeNumSignBits(
RHS) > NumHalfBits) {
3753 SDValue LHSL, LHSH, RHSL, RHSH;
3754 GetExpandedInteger(
LHS, LHSL, LHSH);
3755 GetExpandedInteger(
RHS, RHSL, RHSH);
3758 Lo = DAG.getNode(
N->getOpcode(),
DL, NVT, LHSL, RHSL);
3760 DAG.getShiftAmountConstant(NumHalfBits - 1, NVT,
DL));
3768 SDValue LHSL, LHSH, RHSL, RHSH;
3769 GetExpandedInteger(
LHS, LHSL, LHSH);
3770 GetExpandedInteger(
RHS, RHSL, RHSH);
3772 EVT CCT = getSetCCResultType(NVT);
3775 DAG.getSetCC(
DL, CCT, LHSH, DAG.getConstant(0,
DL, NVT),
ISD::SETLT);
3777 Lo = DAG.getSelect(
DL, NVT, HiNeg, LHSL, DAG.getAllOnesConstant(
DL, NVT));
3779 Lo = DAG.getSelect(
DL, NVT, HiNeg, DAG.getConstant(0,
DL, NVT), LHSL);
3781 Hi = DAG.getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3785 const APInt *RHSVal =
nullptr;
3787 RHSVal = &RHSConst->getAPIntValue();
3794 SDValue LHSL, LHSH, RHSL, RHSH;
3795 GetExpandedInteger(
LHS, LHSL, LHSH);
3796 GetExpandedInteger(
RHS, RHSL, RHSH);
3798 EVT CCT = getSetCCResultType(NVT);
3804 Hi = DAG.getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3807 SDValue IsHiLeft = DAG.getSetCC(
DL, CCT, LHSH, RHSH, CondC);
3811 SDValue LoCmp = DAG.getSelect(
DL, NVT, IsHiLeft, LHSL, RHSL);
3814 SDValue LoMinMax = DAG.getNode(LoOpc,
DL, NVT, {LHSL, RHSL});
3816 Lo = DAG.getSelect(
DL, NVT, IsHiEq, LoMinMax, LoCmp);
3823 switch (
N->getOpcode()) {
3850 EVT VT =
N->getValueType(0);
3851 EVT CCT = getSetCCResultType(VT);
3854 SplitInteger(Result,
Lo,
Hi);
3858 SDValue ExpandedCMP = TLI.expandCMP(
N, DAG);
3859 SplitInteger(ExpandedCMP,
Lo,
Hi);
3862void DAGTypeLegalizer::ExpandIntRes_ADDSUB(
SDNode *
N,
3866 SDValue LHSL, LHSH, RHSL, RHSH;
3867 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3868 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3871 SDValue LoOps[2] = { LHSL, RHSL };
3872 SDValue HiOps[3] = { LHSH, RHSH };
3874 bool HasOpCarry = TLI.isOperationLegalOrCustom(
3876 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3878 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3882 Hi = DAG.computeKnownBits(HiOps[2]).isZero()
3888 Hi = DAG.computeKnownBits(HiOps[2]).isZero()
3901 TLI.isOperationLegalOrCustom(
N->getOpcode() ==
ISD::ADD ?
3903 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3906 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
3920 TLI.isOperationLegalOrCustom(
N->getOpcode() ==
ISD::ADD ?
3922 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3926 EVT OvfVT = getSetCCResultType(NVT);
3927 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
3942 OVF = DAG.
getNode(
ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
3945 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
3949 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
3961 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT),
Lo,
3965 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3968 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3971 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT),
Lo, LoOps[0],
3976 Carry = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3978 Carry = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3979 DAG.getConstant(0, dl, NVT));
3982 Hi = DAG.getNode(
ISD::SUB, dl, NVT, HiOps[0], Carry);
3991 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].
getValueType()),
3996 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3998 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3999 DAG.getConstant(0, dl, NVT));
4005void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(
SDNode *
N,
4008 SDValue LHSL, LHSH, RHSL, RHSH;
4010 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
4011 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
4012 SDVTList VTList = DAG.getVTList(LHSL.
getValueType(), MVT::Glue);
4013 SDValue LoOps[2] = { LHSL, RHSL };
4014 SDValue HiOps[3] = { LHSH, RHSH };
4028 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
4031void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(
SDNode *
N,
4034 SDValue LHSL, LHSH, RHSL, RHSH;
4036 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
4037 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
4038 SDVTList VTList = DAG.getVTList(LHSL.
getValueType(), MVT::Glue);
4040 SDValue HiOps[3] = { LHSH, RHSH };
4042 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
4044 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
4048 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
4051void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(
SDNode *
N,
4059 unsigned CarryOp, NoCarryOp;
4061 switch(
N->getOpcode()) {
4076 bool HasCarryOp = TLI.isOperationLegalOrCustom(
4077 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(),
LHS.getValueType()));
4081 SDValue LHSL, LHSH, RHSL, RHSH;
4082 GetExpandedInteger(
LHS, LHSL, LHSH);
4083 GetExpandedInteger(
RHS, RHSL, RHSH);
4084 SDVTList VTList = DAG.getVTList(LHSL.
getValueType(),
N->getValueType(1));
4085 SDValue LoOps[2] = { LHSL, RHSL };
4086 SDValue HiOps[3] = { LHSH, RHSH };
4088 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
4090 Hi = DAG.
getNode(CarryOp, dl, VTList, HiOps);
4097 SplitInteger(Sum,
Lo,
Hi);
4103 Ovf = DAG.getSetCC(dl,
N->getValueType(1),
Or,
4104 DAG.getConstant(0, dl,
Lo.getValueType()),
ISD::SETEQ);
4108 DAG.getSetCC(dl,
N->getValueType(1),
LHS,
4113 Ovf = DAG.getSetCC(dl,
N->getValueType(1), Sum,
LHS,
Cond);
4119 ReplaceValueWith(
SDValue(
N, 1), Ovf);
4125 SDValue LHSL, LHSH, RHSL, RHSH;
4127 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
4128 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
4129 SDVTList VTList = DAG.getVTList(LHSL.
getValueType(),
N->getValueType(1));
4133 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
4135 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
4139 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
4142void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(
SDNode *
N,
4145 SDValue LHSL, LHSH, RHSL, RHSH;
4147 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
4148 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
4149 SDVTList VTList = DAG.getVTList(LHSL.
getValueType(),
N->getValueType(1));
4154 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL,
N->
getOperand(2) });
4155 Hi = DAG.getNode(
N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
4159 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
4162void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(
SDNode *
N,
4164 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
4167 if (
Op.getValueType().bitsLE(NVT)) {
4170 Hi = DAG.getUNDEF(NVT);
4174 assert(getTypeAction(
Op.getValueType()) ==
4176 "Only know how to promote this result!");
4179 "Operand over promoted?");
4181 SplitInteger(Res,
Lo,
Hi);
4185void DAGTypeLegalizer::ExpandIntRes_AssertSext(
SDNode *
N,
4188 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4189 EVT NVT =
Lo.getValueType();
4194 if (NVTBits < EVTBits) {
4197 EVTBits - NVTBits)));
4202 DAG.getShiftAmountConstant(NVTBits - 1, NVT, dl));
4206void DAGTypeLegalizer::ExpandIntRes_AssertZext(
SDNode *
N,
4209 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4210 EVT NVT =
Lo.getValueType();
4215 if (NVTBits < EVTBits) {
4218 EVTBits - NVTBits)));
4222 Hi = DAG.getConstant(0, dl, NVT);
4226void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(
SDNode *
N,
4229 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
4234void DAGTypeLegalizer::ExpandIntRes_BSWAP(
SDNode *
N,
4237 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
4246 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4247 EVT NVT =
Lo.getValueType();
4250 Hi = DAG.getConstant(0, dl, NVT);
4253void DAGTypeLegalizer::ExpandIntRes_Constant(
SDNode *
N,
4255 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
4258 const APInt &Cst =
Constant->getAPIntValue();
4259 bool IsTarget =
Constant->isTargetOpcode();
4260 bool IsOpaque =
Constant->isOpaque();
4262 Lo = DAG.getConstant(Cst.
trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
4263 Hi = DAG.getConstant(Cst.
lshr(NBitWidth).
trunc(NBitWidth), dl, NVT, IsTarget,
4271 GetExpandedInteger(N0,
Lo,
Hi);
4272 EVT NVT =
Lo.getValueType();
4277 unsigned NumSignBits = DAG.ComputeNumSignBits(N0);
4282 Lo = DAG.getNode(AbsOpc, dl, NVT,
Lo);
4283 Hi = DAG.getConstant(0, dl, NVT);
4293 bool HasSubCarry = TLI.isOperationLegalOrCustom(
4298 DAG.getShiftAmountConstant(NVT.
getSizeInBits() - 1, NVT, dl));
4299 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
4308 EVT VT =
N->getValueType(0);
4310 DAG.getConstant(0, dl, VT), N0);
4312 SplitInteger(Neg, NegLo, NegHi);
4314 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
Hi,
4316 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo,
Lo);
4317 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi,
Hi);
4320void DAGTypeLegalizer::ExpandIntRes_CTLZ(
SDNode *
N,
4324 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4325 EVT NVT =
Lo.getValueType();
4327 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT),
Hi,
4330 SDValue LoLZ = DAG.getNode(
N->getOpcode(), dl, NVT,
Lo);
4333 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
4334 DAG.getNode(
ISD::ADD, dl, NVT, LoLZ,
4337 Hi = DAG.getConstant(0, dl, NVT);
4345 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4346 EVT NVT =
Lo.getValueType();
4349 SDValue Constant0 = DAG.getConstant(0, dl, NVT);
4350 SDValue ConstantBWM1 = DAG.getConstant(NVTBits - 1, dl, NVT);
4353 SDValue IsAllSignBits = DAG.getSetCC(dl, getSetCCResultType(NVT), HiCTLS,
4356 DAG.getSetCC(dl, getSetCCResultType(NVT),
Hi, Constant0,
ISD::SETLT);
4358 DAG.getSelect(dl, NVT, IsNegative, DAG.getNOT(dl,
Lo, NVT),
Lo);
4360 Lo = DAG.getSelect(dl, NVT, IsAllSignBits,
4361 DAG.getNode(
ISD::ADD, dl, NVT, LoCLZ, ConstantBWM1),
4363 Hi = DAG.getConstant(0, dl, NVT);
4368 SplitInteger(Result,
Lo,
Hi);
4373 EVT VT =
N->getValueType(0);
4378 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4379 "LibCall explicitly requested, but not available");
4381 if (RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC)) {
4382 TargetLowering::MakeLibCallOptions CallOptions;
4386 TLI.makeLibCall(DAG, LCImpl, IntVT,
Op, CallOptions,
DL).first;
4387 SplitInteger(DAG.getSExtOrTrunc(Res,
DL, VT),
Lo,
Hi);
4395 GetExpandedInteger(
Op,
Lo,
Hi);
4396 EVT NVT =
Lo.getValueType();
4399 Hi = DAG.getConstant(0,
DL, NVT);
4402void DAGTypeLegalizer::ExpandIntRes_CTTZ(
SDNode *
N,
4406 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4407 EVT NVT =
Lo.getValueType();
4409 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT),
Lo,
4415 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
4416 DAG.getNode(
ISD::ADD, dl, NVT, HiLZ,
4419 Hi = DAG.getConstant(0, dl, NVT);
4425 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
4432 DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl));
4436 ReplaceValueWith(
SDValue(
N, 1), Chain);
4444 Chain =
Op.getValue(1);
4453 EVT VT =
N->getValueType(0);
4457 bool IsStrict =
N->isStrictFPOpcode();
4459 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
4464 EVT SrcVT =
Op.getValueType();
4465 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4472 (TLI.isTypeLegal(SrcVT) || (SoftPromoted && TLI.isTypeLegal(MVT::f32)))) {
4473 unsigned NewOpc = TLI.getPreferredFPToIntOpcode(
N->getOpcode(), SrcVT, NVT);
4475 Lo = DAG.getNode(NewOpc, dl, {NVT, MVT::Other}, {Chain,
Op});
4476 ReplaceValueWith(
SDValue(
N, 1),
Lo.getValue(1));
4478 Lo = DAG.getNode(NewOpc, dl, NVT,
Op);
4481 DAG.getShiftAmountConstant(
4483 : DAG.getConstant(0, dl, NVT);
4489 Op.getValueType() == MVT::bf16) {
4495 EVT OpVT =
Op.getValueType();
4499 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-xint conversion!");
4500 TargetLowering::MakeLibCallOptions CallOptions;
4505 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT,
Op,
4506 CallOptions, dl, Chain);
4507 SplitInteger(Tmp.first,
Lo,
Hi);
4510 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
4515 SDValue Res = TLI.expandFP_TO_INT_SAT(
N, DAG);
4516 SplitInteger(Res,
Lo,
Hi);
4522 bool IsStrict =
N->isStrictFPOpcode();
4523 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
4526 EVT VT =
Op.getValueType();
4528 if (VT == MVT::f16) {
4534 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4537 LC = RTLIB::getLROUND(VT);
4538 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lround input type!");
4541 LC = RTLIB::getLRINT(VT);
4542 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lrint input type!");
4545 LC = RTLIB::getLLROUND(VT);
4546 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llround input type!");
4549 LC = RTLIB::getLLRINT(VT);
4550 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llrint input type!");
4554 EVT RetVT =
N->getValueType(0);
4556 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
4557 if (LCImpl == RTLIB::Unsupported) {
4558 DAG.getContext()->emitError(Twine(
"no libcall available for ") +
4559 N->getOperationName(&DAG));
4561 SplitInteger(Poison,
Lo,
Hi);
4562 if (
N->isStrictFPOpcode())
4563 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
4567 TargetLowering::MakeLibCallOptions CallOptions;
4569 std::pair<SDValue, SDValue> Tmp =
4570 TLI.makeLibCall(DAG, LCImpl, RetVT,
Op, CallOptions, dl, Chain);
4571 SplitInteger(Tmp.first,
Lo,
Hi);
4573 if (
N->isStrictFPOpcode())
4574 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
4577void DAGTypeLegalizer::ExpandIntRes_LOAD(
LoadSDNode *
N,
4579 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_LOAD?");
4582 ExpandRes_NormalLoad(
N,
Lo,
Hi);
4588 EVT VT =
N->getValueType(0);
4589 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4594 MMOMetadata
Metadata =
N->getMMOMetadataForSubAccess();
4599 if (
N->getMemoryVT().bitsLE(NVT)) {
4600 EVT MemVT =
N->getMemoryVT();
4602 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
N->getPointerInfo(), MemVT,
4611 unsigned LoSize =
Lo.getValueSizeInBits();
4613 DAG.getShiftAmountConstant(LoSize - 1, NVT, dl));
4616 Hi = DAG.getConstant(0, dl, NVT);
4620 Hi = DAG.getUNDEF(NVT);
4622 }
else if (DAG.getDataLayout().isLittleEndian()) {
4624 Lo = DAG.getLoad(NVT, dl, Ch, Ptr,
N->getPointerInfo(),
N->getBaseAlign(),
4627 unsigned ExcessBits =
4634 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
4635 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
4645 EVT MemVT =
N->getMemoryVT();
4648 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4651 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
N->getPointerInfo(),
4660 N->getPointerInfo().getWithOffset(IncrementSize),
4674 DAG.getShiftAmountConstant(ExcessBits, NVT, dl)));
4678 DAG.getShiftAmountConstant(
4685 ReplaceValueWith(
SDValue(
N, 1), Ch);
4688void DAGTypeLegalizer::ExpandIntRes_Logical(
SDNode *
N,
4692 GetExpandedInteger(
N->getOperand(0), LL, LH);
4693 GetExpandedInteger(
N->getOperand(1), RL, RH);
4697 Flags.setDisjoint(
N->getFlags().hasDisjoint());
4699 Lo = DAG.getNode(
N->getOpcode(), dl, LL.
getValueType(), LL, RL, Flags);
4700 Hi = DAG.getNode(
N->getOpcode(), dl, LL.
getValueType(), LH, RH, Flags);
4703void DAGTypeLegalizer::ExpandIntRes_MUL(
SDNode *
N,
4705 EVT VT =
N->getValueType(0);
4706 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4710 GetExpandedInteger(
N->getOperand(0), LL, LH);
4711 GetExpandedInteger(
N->getOperand(1), RL, RH);
4713 if (TLI.expandMUL(
N,
Lo,
Hi, NVT, DAG,
4720 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
4721 if (LCImpl == RTLIB::Unsupported) {
4724 TLI.forceExpandMultiply(DAG, dl,
false,
Lo,
Hi, LL, RL, LH, RH);
4730 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4731 TargetLowering::MakeLibCallOptions CallOptions;
4733 SplitInteger(TLI.makeLibCall(DAG, LCImpl, VT,
Ops, CallOptions, dl).first,
Lo,
4740 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
4741 SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
4742 SDValue R = DAG.getNode(
N->getOpcode(),
DL, VTs,
N->getOperand(0));
4745 ReplaceValueWith(
SDValue(
N, 1),
R.getValue(2));
4750 SplitInteger(Result,
Lo,
Hi);
4756 SplitInteger(Result,
Lo,
Hi);
4762 SplitInteger(Result,
Lo,
Hi);
4773 EVT VT =
N->getValueType(0);
4777 uint64_t Scale =
N->getConstantOperandVal(2);
4789 EVT BoolVT = getSetCCResultType(VT);
4791 Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT),
LHS,
RHS);
4797 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
4798 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
4804 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
4805 Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
4810 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
4811 Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
4814 SplitInteger(Result,
Lo,
Hi);
4820 assert(Scale <= VTSize &&
"Scale can't be larger than the value type size.");
4822 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4824 GetExpandedInteger(
LHS, LL, LH);
4825 GetExpandedInteger(
RHS, RL, RH);
4829 if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl,
LHS,
RHS, Result, NVT, DAG,
4836 TLI.forceExpandWideMUL(DAG, dl,
Signed,
LHS,
RHS, LoTmp, HiTmp);
4837 SplitInteger(LoTmp, Result[0], Result[1]);
4838 SplitInteger(HiTmp, Result[2], Result[3]);
4840 assert(
Result.size() == 4 &&
"Unexpected number of partlets in the result");
4843 assert((VTSize == NVTSize * 2) &&
"Expected the new value type to be half "
4844 "the size of the current value type");
4866 if (Scale % NVTSize) {
4867 SDValue ShiftAmount = DAG.getShiftAmountConstant(Scale % NVTSize, NVT, dl);
4870 Hi = DAG.getNode(
ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
4882 if (Scale == VTSize)
4901 SDValue NVTZero = DAG.getConstant(0, dl, NVT);
4902 SDValue NVTNeg1 = DAG.getAllOnesConstant(dl, NVT);
4903 EVT BoolNVT = getSetCCResultType(NVT);
4906 if (Scale < NVTSize) {
4909 DAG.getNode(
ISD::SRL, dl, NVT, ResultHL,
4910 DAG.getShiftAmountConstant(Scale, NVT, dl));
4911 SDValue Tmp = DAG.getNode(
ISD::OR, dl, NVT, HLAdjusted, ResultHH);
4912 SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero,
ISD::SETNE);
4913 }
else if (Scale == NVTSize) {
4915 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero,
ISD::SETNE);
4916 }
else if (Scale < VTSize) {
4919 DAG.getNode(
ISD::SRL, dl, NVT, ResultHH,
4920 DAG.getShiftAmountConstant(Scale - NVTSize, NVT, dl));
4921 SatMax = DAG.getSetCC(dl, BoolNVT, HHAdjusted, NVTZero,
ISD::SETNE);
4924 "(and saturation can't happen with Scale==VTSize).");
4926 Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1,
Hi);
4927 Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1,
Lo);
4931 if (Scale < NVTSize) {
4936 unsigned OverflowBits = VTSize - Scale + 1;
4937 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4938 "Extent of overflow bits must start within HL");
4939 SDValue HLHiMask = DAG.getConstant(
4941 SDValue HLLoMask = DAG.getConstant(
4948 DAG.getNode(
ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
4954 DAG.getNode(
ISD::AND, dl, BoolNVT, HHEQ, HLULT));
4955 }
else if (Scale == NVTSize) {
4961 DAG.getNode(
ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
4967 DAG.getNode(
ISD::AND, dl, BoolNVT, HHEQ, HLPos));
4968 }
else if (Scale < VTSize) {
4971 unsigned OverflowBits = VTSize - Scale + 1;
4972 SDValue HHHiMask = DAG.getConstant(
4974 SDValue HHLoMask = DAG.getConstant(
4976 SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask,
ISD::SETGT);
4977 SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask,
ISD::SETLT);
4984 Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT),
Hi);
4985 Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT),
Lo);
4988 Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT),
Hi);
4989 Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero,
Lo);
4996 SDValue Res = TLI.expandFixedPointDiv(
N->getOpcode(), dl,
N->getOperand(0),
4998 N->getConstantOperandVal(2), DAG);
5002 N->getConstantOperandVal(2), TLI, DAG);
5003 SplitInteger(Res,
Lo,
Hi);
5006void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(
SDNode *Node,
5009 "Node has unexpected Opcode");
5019 bool HasCarryOp = TLI.isOperationLegalOrCustom(
5020 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(),
LHS.getValueType()));
5024 SDValue LHSL, LHSH, RHSL, RHSH;
5025 GetExpandedInteger(
LHS, LHSL, LHSH);
5026 GetExpandedInteger(
RHS, RHSL, RHSH);
5027 SDVTList VTList = DAG.getVTList(LHSL.
getValueType(),
Node->getValueType(1));
5030 Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH,
Lo.
getValue(1) });
5039 SplitInteger(Sum,
Lo,
Hi);
5064 EVT VT =
LHS.getValueType();
5067 SignsMatch = DAG.getNOT(dl, SignsMatch, VT);
5071 EVT OType =
Node->getValueType(1);
5072 Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT),
ISD::SETLT);
5076 ReplaceValueWith(
SDValue(Node, 1), Ovf);
5079void DAGTypeLegalizer::ExpandIntRes_SDIV(
SDNode *
N,
5081 EVT VT =
N->getValueType(0);
5083 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5092 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SDIV!");
5094 TargetLowering::MakeLibCallOptions CallOptions;
5096 SplitInteger(TLI.makeLibCall(DAG, LC, VT,
Ops, CallOptions, dl).first,
Lo,
Hi);
5099void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(
SDNode *
N,
SDValue &
Lo,
5102 SDValue Shiftee =
N->getOperand(0);
5109 LoadVT = TLI.getTypeToTransformTo(*DAG.getContext(), LoadVT);
5110 }
while (!TLI.isTypeLegal(LoadVT));
5115 "Shifting unit is not a a power of two!");
5117 const bool IsOneStepShift =
5118 DAG.computeKnownBits(ShAmt).countMinTrailingZeros() >=
5123 if (!IsOneStepShift)
5124 ShAmt = DAG.getFreeze(ShAmt);
5127 assert(VTBitWidth % 8 == 0 &&
"Shifting a not byte multiple value?");
5128 unsigned VTByteWidth = VTBitWidth / 8;
5130 "Shiftee type size is not a power of two!");
5131 unsigned StackSlotByteWidth = 2 * VTByteWidth;
5132 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
5137 Align StackAlign = DAG.getReducedAlign(StackSlotVT,
false);
5139 DAG.CreateStackTemporary(StackSlotVT.
getStoreSize(), StackAlign);
5140 EVT PtrTy =
StackPtr.getValueType();
5141 SDValue Ch = DAG.getEntryNode();
5144 DAG.getMachineFunction(),
5150 unsigned WideningOpc =
5152 Init = DAG.
getNode(WideningOpc, dl, StackSlotVT, Shiftee);
5155 SDValue AllZeros = DAG.getConstant(0, dl, VT);
5159 Ch = DAG.getStore(Ch, dl, Init, StackPtr, StackPtrInfo, StackAlign);
5165 Flags.setExact(IsOneStepShift);
5168 DAG.getConstant(
Log2_32(ShiftUnitInBits), dl, ShAmtVT), Flags);
5170 DAG.getNode(
ISD::SHL, dl, ShAmtVT, SrlTmp,
5171 DAG.getConstant(
Log2_32(ShiftUnitInBits), dl, ShAmtVT));
5174 DAG.getNode(
ISD::SRL, dl, ShAmtVT, BitOffset,
5179 DAG.getConstant(VTByteWidth - 1, dl, ShAmtVT));
5186 if (DAG.getDataLayout().isBigEndian())
5187 WillIndexUpwards = !WillIndexUpwards;
5190 if (WillIndexUpwards) {
5193 AdjStackPtr = DAG.getMemBasePlusOffset(
5194 StackPtr, DAG.getConstant(VTByteWidth, dl, PtrTy), dl);
5195 ByteOffset = DAG.getNegative(ByteOffset, dl, ShAmtVT);
5199 ByteOffset = DAG.getSExtOrTrunc(ByteOffset, dl, PtrTy);
5200 AdjStackPtr = DAG.getMemBasePlusOffset(AdjStackPtr, ByteOffset, dl);
5204 DAG.getLoad(VT, dl, Ch, AdjStackPtr,
5209 if (!IsOneStepShift) {
5211 DAG.getNode(
ISD::AND, dl, ShAmtVT, ShAmt,
5212 DAG.getConstant(ShiftUnitInBits - 1, dl, ShAmtVT));
5213 Res = DAG.
getNode(
N->getOpcode(), dl, VT, Res, ShAmtRem);
5217 SplitInteger(Res,
Lo,
Hi);
5220void DAGTypeLegalizer::ExpandIntRes_Shift(
SDNode *
N,
5222 EVT VT =
N->getValueType(0);
5223 unsigned Opc =
N->getOpcode();
5229 return ExpandShiftByConstant(
N, CN->getAPIntValue(),
Lo,
Hi);
5233 if (ExpandShiftWithKnownAmountBit(
N,
Lo,
Hi))
5250 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
5252 const bool LegalOrCustom =
5256 unsigned ExpansionFactor = 1;
5258 for (EVT TmpVT = NVT;;) {
5259 EVT NewTMPVT = TLI.getTypeToTransformTo(*DAG.getContext(), TmpVT);
5260 if (NewTMPVT == TmpVT)
5267 TLI.preferredShiftLegalizationStrategy(DAG,
N, ExpansionFactor);
5270 return ExpandIntRes_ShiftThroughStack(
N,
Lo,
Hi);
5272 if (LegalOrCustom &&
5276 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
5282 SDValue ShiftOp =
N->getOperand(1);
5283 EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
5285 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
5288 Lo = DAG.
getNode(PartsOpc, dl, DAG.getVTList(VT, VT),
Ops);
5289 Hi =
Lo.getValue(1);
5294 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5308 if (RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(LC)) {
5311 SDValue ShAmt = DAG.getZExtOrTrunc(
N->getOperand(1), dl, ShAmtTy);
5313 TargetLowering::MakeLibCallOptions CallOptions;
5316 TLI.makeLibCall(DAG, LibcallImpl, VT,
Ops, CallOptions, dl).first,
Lo,
5321 if (!ExpandShiftWithUnknownAmountBit(
N,
Lo,
Hi))
5325void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(
SDNode *
N,
5327 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
5330 if (
Op.getValueType().bitsLE(NVT)) {
5336 DAG.getShiftAmountConstant(LoSize - 1, NVT, dl));
5340 assert(getTypeAction(
Op.getValueType()) ==
5342 "Only know how to promote this result!");
5345 "Operand over promoted?");
5347 SplitInteger(Res,
Lo,
Hi);
5355void DAGTypeLegalizer::
5358 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5361 if (EVT.
bitsLE(
Lo.getValueType())) {
5369 DAG.getShiftAmountConstant(
Hi.getValueSizeInBits() - 1,
5370 Hi.getValueType(), dl));
5381void DAGTypeLegalizer::ExpandIntRes_SREM(
SDNode *
N,
5383 EVT VT =
N->getValueType(0);
5385 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5394 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SREM!");
5396 TargetLowering::MakeLibCallOptions CallOptions;
5398 SplitInteger(TLI.makeLibCall(DAG, LC, VT,
Ops, CallOptions, dl).first,
Lo,
Hi);
5401void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(
SDNode *
N,
5403 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
5413void DAGTypeLegalizer::ExpandIntRes_XMULO(
SDNode *
N,
5415 EVT VT =
N->getValueType(0);
5434 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
5435 GetExpandedInteger(
LHS, LHSLow, LHSHigh);
5436 GetExpandedInteger(
RHS, RHSLow, RHSHigh);
5438 EVT BitVT =
N->getValueType(1);
5439 SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT);
5441 SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
5443 DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero,
ISD::SETNE),
5444 DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero,
ISD::SETNE));
5463 SplitInteger(Three,
Lo,
Hi);
5467 ReplaceValueWith(
SDValue(
N, 1), Overflow);
5472 EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
5477 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
5481 if (LCImpl == RTLIB::Unsupported ||
5483 DAG.getMachineFunction().getName()) {
5486 TLI.forceExpandWideMUL(DAG, dl,
true,
N->getOperand(0),
5487 N->getOperand(1), MulLo, MulHi);
5492 DAG.getSetCC(dl,
N->getValueType(1), MulHi, SRA,
ISD::SETNE);
5493 SplitInteger(MulLo,
Lo,
Hi);
5494 ReplaceValueWith(
SDValue(
N, 1), Overflow);
5498 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
5501 DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
5502 MachinePointerInfo());
5506 EVT ArgVT =
Op.getValueType();
5508 TargetLowering::ArgListEntry
Entry(
Op, ArgTy);
5509 Entry.IsSExt =
true;
5510 Entry.IsZExt =
false;
5511 Args.push_back(Entry);
5515 TargetLowering::ArgListEntry
Entry(
5517 Entry.IsSExt =
true;
5518 Entry.IsZExt =
false;
5519 Args.push_back(Entry);
5521 SDValue Func = DAG.getExternalSymbol(LCImpl, PtrVT);
5523 TargetLowering::CallLoweringInfo CLI(DAG);
5526 .setLibCallee(DAG.getLibcalls().getLibcallImplCallingConv(LCImpl), RetTy,
5527 Func, std::move(Args))
5530 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
5532 SplitInteger(CallInfo.first,
Lo,
Hi);
5534 DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
5535 SDValue Ofl = DAG.getSetCC(dl,
N->getValueType(1), Temp2,
5536 DAG.getConstant(0, dl, PtrVT),
5539 ReplaceValueWith(
SDValue(
N, 1), Ofl);
5542void DAGTypeLegalizer::ExpandIntRes_UDIV(
SDNode *
N,
5544 EVT VT =
N->getValueType(0);
5546 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5556 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
5558 if (isTypeLegal(NVT)) {
5560 GetExpandedInteger(
N->getOperand(0), InL, InH);
5562 if (TLI.expandDIVREMByConstant(
N, Result, NVT, DAG, InL, InH)) {
5571 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UDIV!");
5573 TargetLowering::MakeLibCallOptions CallOptions;
5574 SplitInteger(TLI.makeLibCall(DAG, LC, VT,
Ops, CallOptions, dl).first,
Lo,
Hi);
5577void DAGTypeLegalizer::ExpandIntRes_UREM(
SDNode *
N,
5579 EVT VT =
N->getValueType(0);
5581 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5591 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
5593 if (isTypeLegal(NVT)) {
5595 GetExpandedInteger(
N->getOperand(0), InL, InH);
5597 if (TLI.expandDIVREMByConstant(
N, Result, NVT, DAG, InL, InH)) {
5606 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UREM!");
5608 TargetLowering::MakeLibCallOptions CallOptions;
5609 SplitInteger(TLI.makeLibCall(DAG, LC, VT,
Ops, CallOptions, dl).first,
Lo,
Hi);
5612void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(
SDNode *
N,
5614 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
5617 if (
Op.getValueType().bitsLE(NVT)) {
5620 Hi = DAG.getConstant(0, dl, NVT);
5624 assert(getTypeAction(
Op.getValueType()) ==
5626 "Only know how to promote this result!");
5629 "Operand over promoted?");
5631 SplitInteger(Res,
Lo,
Hi);
5633 Hi = DAG.getZeroExtendInReg(
Hi, dl,
5639void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(
SDNode *
N,
5643 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
5654void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(
SDNode *
N,
5658 SDValue Res = TLI.expandVecReduce(
N, DAG);
5659 SplitInteger(Res,
Lo,
Hi);
5662void DAGTypeLegalizer::ExpandIntRes_Rotate(
SDNode *
N,
5668 N->getOperand(0),
N->getOperand(1));
5669 SplitInteger(Res,
Lo,
Hi);
5676 GetExpandedInteger(
N->getOperand(0), In3, In4);
5677 GetExpandedInteger(
N->getOperand(1), In1, In2);
5681 unsigned Opc =
N->getOpcode();
5684 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5689 DAG.getConstant(HalfVTBits,
DL, ShAmtVT));
5691 DAG.getSetCC(
DL, ShAmtCCVT, AndNode, DAG.getConstant(0,
DL, ShAmtVT),
5695 EVT NewShAmtVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
5696 SDValue NewShAmt = DAG.getAnyExtOrTrunc(ShAmt,
DL, NewShAmtVT);
5701 Lo = DAG.getNode(
Opc,
DL, HalfVT, Select2, Select1, NewShAmt);
5702 Hi = DAG.getNode(
Opc,
DL, HalfVT, Select3, Select2, NewShAmt);
5707 SDValue Res = TLI.expandCLMUL(
N, DAG);
5708 return SplitInteger(Res,
Lo,
Hi);
5712 GetExpandedInteger(
N->getOperand(0), LL, LH);
5713 GetExpandedInteger(
N->getOperand(1), RL, RH);
5727 Hi = DAG.getNode(
ISD::XOR,
DL, HalfVT, LoH, HiLoCross);
5731 SDValue Res = TLI.expandPEXT(
N, DAG);
5732 SplitInteger(Res,
Lo,
Hi);
5736 SDValue Res = TLI.expandPDEP(
N, DAG);
5737 SplitInteger(Res,
Lo,
Hi);
5741 SDValue Res = TLI.expandMULH(
N, DAG);
5742 SplitInteger(Res,
Lo,
Hi);
5747 EVT VT =
N->getValueType(0);
5754 SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One);
5757 SplitInteger(Res,
Lo,
Hi);
5764 "cannot use llvm.read_register with illegal type", Fn,
N->getDebugLoc()));
5765 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
5767 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
5768 Lo = DAG.getPOISON(LoVT);
5769 Hi = DAG.getPOISON(HiVT);
5776 EVT VT =
N->getSimpleValueType(0);
5777 EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
5779 "VectorIdxTy should be smaller than type to be expanded?");
5783 SplitInteger(Res,
Lo,
Hi);
5794bool DAGTypeLegalizer::ExpandIntegerOperand(
SDNode *
N,
unsigned OpNo) {
5798 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
5801 switch (
N->getOpcode()) {
5804 dbgs() <<
"ExpandIntegerOperand Op #" << OpNo <<
": ";
5805 N->dump(&DAG);
dbgs() <<
"\n";
5810 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(
N);
break;
5814 Res = ExpandOp_FAKE_USE(
N);
5818 Res = TLI.expandLoopDependenceMask(
N, DAG);
5824 case ISD::SETCC: Res = ExpandIntOp_SETCC(
N);
break;
5837 case ISD::ROTR: Res = ExpandIntOp_Shift(
N);
break;
5842 case ISD::UCMP: Res = ExpandIntOp_CMP(
N);
break;
5846 Res = ExpandIntOp_STACKMAP(
N, OpNo);
5849 Res = ExpandIntOp_PATCHPOINT(
N, OpNo);
5851 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5852 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5853 Res = ExpandIntOp_VP_STRIDED(
N, OpNo);
5856 Res = ExpandIntOp_WRITE_REGISTER(
N, OpNo);
5861 if (!Res.
getNode())
return false;
5869 "Invalid operand expansion");
5871 ReplaceValueWith(
SDValue(
N, 0), Res);
5877void DAGTypeLegalizer::IntegerExpandSetCCOperands(
SDValue &NewLHS,
5881 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5882 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5883 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5896 NewRHS = DAG.getConstant(0, dl, NewLHS.
getValueType());
5903 if ((CCCode ==
ISD::SETLT && CST->isZero()) ||
5904 (CCCode ==
ISD::SETGT && CST->isAllOnes())) {
5934 LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.
getValueType()), LHSLo,
5935 RHSLo, LowCC,
false, DagCombineInfo, dl);
5937 LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.
getValueType()), LHSLo,
5940 HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.
getValueType()), LHSHi,
5941 RHSHi, CCCode,
false, DagCombineInfo, dl);
5945 LHSHi, RHSHi, DAG.getCondCode(CCCode));
5954 if ((EqAllowed && (HiCmpC && HiCmpC->
isZero())) ||
5956 ((HiCmpC && HiCmpC->
isOne()) || (LoCmpC && LoCmpC->
isZero())))) {
5965 if (LHSHi == RHSHi) {
5974 EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
5975 bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(
ISD::SETCCCARRY, ExpandVT);
5978 if (HasSETCCCARRY) {
5981 bool FlipOperands =
false;
5998 SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
6002 DAG.getCondCode(CCCode));
6008 NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi,
ISD::SETEQ,
6009 false, DagCombineInfo, dl);
6012 DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi,
ISD::SETEQ);
6013 NewLHS = DAG.getSelect(dl, LoCmp.
getValueType(), NewLHS, LoCmp, HiCmp);
6020 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(
N));
6025 NewRHS = DAG.getConstant(0, SDLoc(
N), NewLHS.
getValueType());
6030 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
6031 DAG.getCondCode(CCCode), NewLHS, NewRHS,
6032 N->getOperand(4)), 0);
6038 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(
N));
6043 NewRHS = DAG.getConstant(0, SDLoc(
N), NewLHS.
getValueType());
6048 return SDValue(DAG.UpdateNodeOperands(
N, NewLHS, NewRHS,
6049 N->getOperand(2),
N->getOperand(3),
6050 DAG.getCondCode(CCCode)), 0);
6056 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(
N));
6061 "Unexpected setcc expansion!");
6067 DAG.UpdateNodeOperands(
N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
6075 SDLoc dl = SDLoc(
N);
6077 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
6078 GetExpandedInteger(
LHS, LHSLo, LHSHi);
6079 GetExpandedInteger(
RHS, RHSLo, RHSHi);
6092 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
6102 GetExpandedInteger(
N->getOperand(1),
Lo,
Hi);
6103 return SDValue(DAG.UpdateNodeOperands(
N,
N->getOperand(0),
Lo), 0);
6107 return TLI.expandCMP(
N, DAG);
6115 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
6116 return SDValue(DAG.UpdateNodeOperands(
N,
Lo), 0);
6120 bool IsStrict =
N->isStrictFPOpcode();
6124 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
6125 EVT DstVT =
N->getValueType(0);
6128 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
6129 "Don't know how to expand this XINT_TO_FP!");
6130 TargetLowering::MakeLibCallOptions CallOptions;
6132 std::pair<SDValue, SDValue> Tmp =
6133 TLI.makeLibCall(DAG, LC, DstVT,
Op, CallOptions, SDLoc(
N), Chain);
6138 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
6139 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
6144 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_STORE?");
6147 return ExpandOp_NormalStore(
N, OpNo);
6150 assert(OpNo == 1 &&
"Can only expand the stored value so far");
6152 EVT VT =
N->getOperand(1).getValueType();
6153 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6157 MMOMetadata
Metadata =
N->getMMOMetadataForSubAccess();
6163 if (
N->getMemoryVT().bitsLE(NVT)) {
6164 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
6165 return DAG.getTruncStore(Ch, dl,
Lo, Ptr,
N->getPointerInfo(),
6166 N->getMemoryVT(),
N->getBaseAlign(), MMOFlags,
6170 if (DAG.getDataLayout().isLittleEndian()) {
6172 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
6174 Lo = DAG.getStore(Ch, dl,
Lo, Ptr,
N->getPointerInfo(),
N->getBaseAlign(),
6177 unsigned ExcessBits =
6184 Hi = DAG.getTruncStore(Ch, dl,
Hi, Ptr,
6185 N->getPointerInfo().getWithOffset(IncrementSize),
6186 NEVT,
N->getBaseAlign(), MMOFlags,
Metadata);
6192 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
6194 EVT ExtVT =
N->getMemoryVT();
6197 unsigned ExcessBits = (EBytes - IncrementSize)*8;
6205 DAG.getShiftAmountConstant(NVT.
getSizeInBits() - ExcessBits, NVT, dl));
6209 DAG.getShiftAmountConstant(ExcessBits, NVT, dl)));
6213 Hi = DAG.getTruncStore(Ch, dl,
Hi, Ptr,
N->getPointerInfo(), HiVT,
6219 Lo = DAG.getTruncStore(Ch, dl,
Lo, Ptr,
6220 N->getPointerInfo().getWithOffset(IncrementSize),
6228 GetExpandedInteger(
N->getOperand(0), InL, InH);
6237 N->getOperand(0),
N->getOperand(2),
N->getOperand(1),
6239 return Swap.getValue(1);
6242SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
6243 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
6244 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
6248 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo],
Hi);
6250 return SDValue(DAG.UpdateNodeOperands(
N, NewOps), 0);
6253SDValue DAGTypeLegalizer::ExpandIntOp_WRITE_REGISTER(
SDNode *
N,
unsigned OpNo) {
6256 "cannot use llvm.write_register with illegal type", Fn,
6259 return N->getOperand(0);
6262SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(
SDNode *
N) {
6265 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6266 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
6267 EVT OutVT =
V0.getValueType();
6269 return DAG.getNode(
N->getOpcode(), dl, OutVT, V0,
V1,
N->getOperand(2));
6275 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6276 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
6277 EVT OutVT =
V0.getValueType();
6282 return DAG.
getNode(
N->getOpcode(), dl, OutVT, NewOps);
6285SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REPEAT(
SDNode *
N) {
6288 EVT OutVT =
N->getValueType(0);
6289 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6294 return DAG.getNode(
N->getOpcode(),
DL, NOutVT,
Op);
6297SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
SDNode *
N) {
6299 unsigned Factor =
N->getNumOperands();
6302 for (
unsigned i = 0; i != Factor; i++)
6303 Ops[i] = GetPromotedInteger(
N->getOperand(i));
6308 for (
unsigned i = 0; i != Factor; i++)
6314SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
6316 EVT OutVT =
N->getValueType(0);
6317 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6318 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6322 SDValue BaseIdx =
N->getOperand(1);
6339 DAG.getConstant(
alignDown(IdxVal, NElts), dl,
6343 DAG.getConstant(IdxVal % NElts, dl, BaseIdx.
getValueType()));
6349 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
6358 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
6362 "Promoted operand has an element type greater than result");
6375 InOp0 = GetPromotedInteger(InOp0);
6382 Ops.reserve(OutNumElems);
6383 for (
unsigned i = 0; i != OutNumElems; ++i) {
6388 N->getOperand(0), Index);
6389 SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
6394 return DAG.getBuildVector(NOutVT, dl,
Ops);
6397SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(
SDNode *
N) {
6398 EVT OutVT =
N->getValueType(0);
6399 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6400 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6412 Vec = GetPromotedInteger(Vec);
6418SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(
SDNode *
N) {
6421 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6422 EVT OutVT =
V0.getValueType();
6427SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(
SDNode *
N) {
6429 EVT VT =
N->getValueType(0);
6434 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6435 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
6436 EVT OutVT =
V0.getValueType();
6438 return DAG.getVectorShuffle(OutVT, dl, V0,
V1, NewMask);
6442 EVT OutVT =
N->getValueType(0);
6443 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6444 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6445 unsigned NumElems =
N->getNumOperands();
6452 Ops.reserve(NumElems);
6453 for (
unsigned i = 0; i != NumElems; ++i) {
6455 EVT OpVT =
Op.getValueType();
6460 if (OpVT.
bitsLT(NOutVTElem)) {
6466 ExtOpc = NOutExtOpc;
6467 Op = DAG.getNode(ExtOpc, dl, NOutVTElem,
Op);
6472 return DAG.getBuildVector(NOutVT, dl,
Ops);
6479 assert(!
N->getOperand(0).getValueType().isVector() &&
6480 "Input must be a scalar");
6482 EVT OutVT =
N->getValueType(0);
6483 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6484 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6488 return DAG.getNode(
N->getOpcode(), dl, NOutVT,
Op);
6493 EVT OutVT =
N->getValueType(0);
6494 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6496 "Type must be promoted to a scalable vector type");
6497 const APInt &StepVal =
N->getConstantOperandAPInt(0);
6498 return DAG.getStepVector(dl, NOutVT,
6502SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(
SDNode *
N) {
6505 EVT OutVT =
N->getValueType(0);
6506 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6507 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6509 unsigned NumOperands =
N->getNumOperands();
6513 EVT OpVT =
N->getOperand(0).getValueType();
6518 "Unhandled legalization type");
6524 for (
unsigned I = 0;
I < NumOperands; ++
I) {
6527 Op = GetPromotedInteger(
Op);
6536 *DAG.getContext(),
Ops[0].getValueType().getVectorElementType());
6537 return DAG.getAnyExtOrTrunc(
6541 unsigned NumElem =
N->getOperand(0).getValueType().getVectorNumElements();
6542 assert(NumElem * NumOperands == NumOutElem &&
6543 "Unexpected number of elements");
6547 for (
unsigned i = 0; i < NumOperands; ++i) {
6550 Op = GetPromotedInteger(
Op);
6551 EVT SclrTy =
Op.getValueType().getVectorElementType();
6552 assert(NumElem ==
Op.getValueType().getVectorNumElements() &&
6553 "Unexpected number of elements");
6555 for (
unsigned j = 0;
j < NumElem; ++
j) {
6557 DAG.getVectorIdxConstant(j, dl));
6558 Ops[i * NumElem +
j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
6562 return DAG.getBuildVector(NOutVT, dl,
Ops);
6565SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(
SDNode *
N) {
6566 EVT VT =
N->getValueType(0);
6567 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6568 assert(NVT.
isVector() &&
"This type must be promoted to a vector type");
6576 if (getTypeAction(
N->getOperand(0).getValueType())
6580 switch(
N->getOpcode()) {
6582 Promoted = SExtPromotedInteger(
N->getOperand(0));
6585 Promoted = ZExtPromotedInteger(
N->getOperand(0));
6588 Promoted = GetPromotedInteger(
N->getOperand(0));
6600 DAG.getVectorIdxConstant(0, dl));
6602 return DAG.getNode(
N->getOpcode(), dl, NVT, Promoted);
6606 return DAG.getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
6609SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(
SDNode *
N) {
6610 EVT VT =
N->getValueType(0);
6611 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6615SDValue DAGTypeLegalizer::PromoteIntRes_GET_ACTIVE_LANE_MASK(
SDNode *
N) {
6616 EVT VT =
N->getValueType(0);
6617 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6621SDValue DAGTypeLegalizer::PromoteIntRes_MASK_BEFOREFIRST(
SDNode *
N) {
6622 EVT VT =
N->getValueType(0);
6623 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6624 SDValue Op = PromoteTargetBoolean(
N->getOperand(0), NVT);
6629 EVT VT =
N->getValueType(0);
6630 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6632 NewOps[2] = PromoteTargetBoolean(
N->getOperand(2), NVT);
6636SDValue DAGTypeLegalizer::PromoteIntRes_PARTIAL_REDUCE_MLA(
SDNode *
N) {
6638 EVT VT =
N->getValueType(0);
6639 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
6640 SDValue ExtAcc = GetPromotedInteger(
N->getOperand(0));
6641 return DAG.getNode(
N->getOpcode(),
DL, NVT, ExtAcc,
N->getOperand(1),
6645SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(
SDNode *
N) {
6646 EVT OutVT =
N->getValueType(0);
6647 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
6648 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6653 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6656 NOutVTElem,
N->getOperand(1));
6658 V0, ConvElem,
N->getOperand(2));
6665 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6666 return DAG.getNode(
N->getOpcode(), dl, NVT,
N->ops());
6674 SDValue Start = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
6675 return DAG.getNode(
N->getOpcode(),
DL,
Start.getValueType(), Start,
6676 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
6680 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6683 assert(
N->getNumValues() == 3 &&
"Expected 3 values for PATCHPOINT");
6684 SDVTList VTList = DAG.getVTList({NVT, MVT::Other, MVT::Glue});
6692 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
6697SDValue DAGTypeLegalizer::PromoteIntRes_READ_REGISTER(
SDNode *
N) {
6700 "cannot use llvm.read_register with illegal type", Fn,
N->getDebugLoc()));
6702 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
N->getValueType(0));
6703 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
6704 return DAG.getPOISON(NVT);
6707SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
6709 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6710 SDValue V1 = DAG.getZExtOrTrunc(
N->getOperand(1), dl,
6711 TLI.getVectorIdxTy(DAG.getDataLayout()));
6713 V0->getValueType(0).getScalarType(), V0,
V1);
6718 return DAG.getAnyExtOrTrunc(Ext, dl,
N->getValueType(0));
6721SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(
SDNode *
N) {
6726 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
6729 V1.getValueType().getVectorElementType(),
6730 V0.getValueType().getVectorElementCount());
6731 V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT);
6733 return DAG.getAnyExtOrTrunc(Ext, dl,
N->getValueType(0));
6742 EVT InVT1 =
V1.getValueType();
6745 TLI.getTypeToTransformTo(*DAG.getContext(), InVT1),
V1);
6746 return DAG.getNode(
N->getOpcode(), dl,
N->getValueType(0), V0, VPromoted);
6749SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
6751 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6752 MVT InVT =
V0.getValueType().getSimpleVT();
6759SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(
SDNode *
N) {
6762 EVT ResVT =
N->getValueType(0);
6763 unsigned NumElems =
N->getNumOperands();
6766 EVT PromotedOpTy = TLI.getTypeToTransformTo(*DAG.getContext(),
6767 N->getOperand(0).getValueType());
6770 if (isTypeLegal(WideVT)) {
6777 return DAG.getAnyExtOrTrunc(WideConcat, dl, ResVT);
6781 SDValue ResVec = DAG.getPOISON(ResVT);
6783 for (
unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
6785 unsigned OpNumElts =
Op.getValueType().getVectorMinNumElements();
6787 DAG.getIntPtrConstant(OpIdx * OpNumElts, dl));
6796 DAG.ExtractVectorElements(GetPromotedInteger(
Op), NewOps);
6797 return DAG.getBuildVector(ResVT, dl, NewOps);
6800SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
6813 for (
unsigned I = 0;
I < OpNo;
I++)
6816 EVT Ty =
Op.getValueType();
6817 SDLoc
DL = SDLoc(
N);
6820 DAG.getTargetConstant(StackMaps::ConstantOp,
DL, MVT::i64));
6828 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6833 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
6839SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
6852 for (
unsigned I = 0;
I < OpNo;
I++)
6855 EVT Ty =
Op.getValueType();
6856 SDLoc
DL = SDLoc(
N);
6859 DAG.getTargetConstant(StackMaps::ConstantOp,
DL, MVT::i64));
6867 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6872 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isSigned(unsigned Opcode)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, unsigned SatW, bool Signed, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, SDLoc DL, SelectionDAG &DAG)
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, unsigned Scale, const TargetLowering &TLI, SelectionDAG &DAG, unsigned SatW=0)
static unsigned getExtendForIntVecReduction(SDNode *N)
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
SmallVector< MachineOperand, 4 > Cond
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
This file describes how to lower LLVM code to machine code.
static LLVM_ABI unsigned int semanticsIntSizeInBits(const fltSemantics &, bool)
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
unsigned getActiveBits() const
Compute the number of active bits in the value.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
unsigned countLeadingOnes() const
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned countTrailingZeros() const
unsigned countLeadingZeros() const
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
unsigned countTrailingOnes() const
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
This is an SDNode representing atomic operations.
const APInt & getValue() const
Return the constant as an APInt value reference.
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
constexpr bool isPowerOf2() const
const Function & getFunction() const
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
unsigned getVectorNumElements() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
Flags
Flags values. These may be or'd together.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(LLVMContext &C)
This constructs an opaque pointer to an object in the default address space (address space zero).
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isStrictFPOpcode()
Test if this node is a strict floating point pseudo-op.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
ISD::CondCode getOperandCondCode(unsigned Num) const
Helper method returns the CondCode of a CondCodeSDNode.
SDNodeFlags getFlags() const
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
void reserve(size_type N)
void push_back(const T &Elt)
This class is used to represent ISD::STORE nodes.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
@ TypeScalarizeScalableVector
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
std::vector< ArgListEntry > ArgListTy
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
This class is used to represent an VP_GATHER node.
This class is used to represent a VP_LOAD node.
This class is used to represent an VP_SCATTER node.
This class is used to represent a VP_STORE node.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS*X will result in a value whose quantity matches our ...
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS*X will result in a value whose quantity matches our own.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ POISON
POISON - A poison node.
@ PARTIAL_REDUCE_SMLA
PARTIAL_REDUCE_[U|S]MLA(Accumulator, Input1, Input2) The partial reduction nodes sign or zero extend ...
@ LOOP_DEPENDENCE_RAW_MASK
@ MASK_BEFOREFIRST
Has one mask vector operand.
@ COND_LOOP
COND_LOOP is a conditional branch to self, used for implementing efficient conditional traps.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ VECTOR_FIND_LAST_ACTIVE
Finds the index of the last active mask element Operands: Mask.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECTOR_SHUFFLE_VAR
VECTOR_SHUFFLE_VAR(VEC, MASK) - Returns VEC permuted by MASK, an integer vector with the same number ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ CLMUL
Carry-less multiplication operations.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ CONVERT_FROM_ARBITRARY_FP
CONVERT_FROM_ARBITRARY_FP - This operator converts from an arbitrary floating-point represented as an...
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ CTLS
Count leading redundant sign bits.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ PATCHPOINT
The llvm.experimental.patchpoint.
@ SMULO
Same for multiplication.
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ MASKED_UDIV
Masked vector arithmetic that returns poison on disabled lanes.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ VECTOR_MATCH
VECTOR_MATCH - this corresponds to the llvm.experimental.vector.match intrinsic.
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ STACKMAP
The llvm.experimental.stackmap intrinsic.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ EXPERIMENTAL_VECTOR_HISTOGRAM
Experimental vector histogram intrinsic Operands: Input Chain, Inc, Mask, Base, Index,...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ CONVERT_TO_ARBITRARY_FP
CONVERT_TO_ARBITRARY_FP - Converts a native FP value to an arbitrary floating-point format,...
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_REPEAT
VECTOR_REPEAT(FIXED_LENGTH_VECTOR) Repeatedly copies the elements of the source fixed-length vector t...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
@ LOOP_DEPENDENCE_WAR_MASK
The llvm.loop.dependence.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUREM(EVT VT)
LLVM_ABI Libcall getSHL(EVT VT)
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSDIV(EVT VT)
LLVM_ABI Libcall getSRL(EVT VT)
LLVM_ABI Libcall getSRA(EVT VT)
LLVM_ABI Libcall getUDIV(EVT VT)
LLVM_ABI Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSREM(EVT VT)
LLVM_ABI Libcall getMUL(EVT VT)
LLVM_ABI AtomicLibcall selectAtomicLibcall(const LibcallLoweringInfo &Libcalls, unsigned Opc, AtomicOrdering Order, MVT VT)
Lower to outline atomic libcall if outline atomics enabled, or to sync libcall otherwise.
LLVM_ABI Libcall getCTPOP(EVT VT)
LLVM_ABI Libcall getMULO(EVT VT)
NodeAddr< NodeBase * > Node
NodeAddr< FuncNode * > Func
Type * getValueType(Value *V, bool ReVec, bool LookThroughCmp)
Returns the "element type" of the given value/instruction V.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Known
Known to have no common set bits.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
@ Success
The lock was released successfully.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
EVT changeVectorElementCount(LLVMContext &Context, ElementCount EC) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element coun...
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static StringRef getLibcallImplName(RTLIB::LibcallImpl CallImpl)
Get the libcall routine name for the specified libcall implementation.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
MakeLibCallOptions & setIsSigned(bool Value=true)