58#define DEBUG_TYPE "legalizedag"
64struct FloatSignAsInt {
87class SelectionDAGLegalize {
99 EVT getSetCCResultType(
EVT VT)
const {
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
166 void getSignAsIntValue(FloatSignAsInt &State,
const SDLoc &
DL,
168 SDValue modifySignAsInt(
const FloatSignAsInt &State,
const SDLoc &
DL,
215 dbgs() <<
" with: "; New->dump(&DAG));
218 "Replacing one node with another that produces a different number "
222 UpdatedNodes->
insert(New);
228 dbgs() <<
" with: "; New->dump(&DAG));
232 UpdatedNodes->
insert(New.getNode());
233 ReplacedNode(Old.getNode());
240 for (
unsigned i = 0, e = Old->
getNumValues(); i != e; ++i) {
251 dbgs() <<
" with: "; New->dump(&DAG));
255 UpdatedNodes->
insert(New.getNode());
256 ReplacedNode(Old.getNode());
266 bool isObjectScalable) {
268 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();
274 ObjectSize, MFI.getObjectAlign(FI));
281SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
286 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
288 assert(NumEltsGrowth &&
"Cannot promote to vector type with fewer elts!");
290 if (NumEltsGrowth == 1)
291 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
294 for (
unsigned i = 0; i != NumMaskElts; ++i) {
296 for (
unsigned j = 0;
j != NumEltsGrowth; ++
j) {
303 assert(NewMask.
size() == NumDestElts &&
"Non-integer NumEltsGrowth?");
304 assert(TLI.isShuffleMaskLegal(NewMask, NVT) &&
"Shuffle not legal?");
305 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
324 assert((VT == MVT::f64 || VT == MVT::f32) &&
"Invalid type expansion");
326 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
336 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
342 TLI.ShouldShrinkFPConstant(OrigVT)) {
345 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
353 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
354 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
363 OrigVT, dl, DAG.getEntryNode(), CPIdx,
371 EVT VT =
CP->getValueType(0);
372 SDValue CPIdx = DAG.getConstantPool(
CP->getConstantIntValue(),
373 TLI.getPointerTy(DAG.getDataLayout()));
374 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
376 VT, dl, DAG.getEntryNode(), CPIdx,
402 for (
unsigned i = 0; i != NumElts; ++i)
403 ShufOps.
push_back(i != InsertPos->getZExtValue() ? i : NumElts);
405 return DAG.getVectorShuffle(Vec.
getValueType(), dl, Vec, ScVec, ShufOps);
408 return ExpandInsertToVectorThroughStack(
Op);
435 TLI.isTypeLegal(MVT::i32)) {
437 bitcastToAPInt().zextOrTrunc(32),
438 SDLoc(CFP), MVT::i32);
439 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
440 ST->getOriginalAlign(), MMOFlags, AAInfo);
446 if (TLI.isTypeLegal(MVT::i64)) {
448 zextOrTrunc(64),
SDLoc(CFP), MVT::i64);
449 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
450 ST->getOriginalAlign(), MMOFlags, AAInfo);
453 if (TLI.isTypeLegal(MVT::i32) && !
ST->isVolatile()) {
460 if (DAG.getDataLayout().isBigEndian())
463 Lo = DAG.getStore(Chain, dl,
Lo,
Ptr,
ST->getPointerInfo(),
464 ST->getOriginalAlign(), MMOFlags, AAInfo);
466 Hi = DAG.getStore(Chain, dl,
Hi,
Ptr,
467 ST->getPointerInfo().getWithOffset(4),
468 ST->getOriginalAlign(), MMOFlags, AAInfo);
477void SelectionDAGLegalize::LegalizeStoreOps(
SDNode *
Node) {
486 if (!
ST->isTruncatingStore()) {
489 ReplaceNode(ST, OptStore);
494 MVT VT =
Value.getSimpleValueType();
495 switch (TLI.getOperationAction(
ISD::STORE, VT)) {
497 case TargetLowering::Legal: {
500 EVT MemVT =
ST->getMemoryVT();
502 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
503 *
ST->getMemOperand())) {
506 ReplaceNode(
SDValue(ST, 0), Result);
511 case TargetLowering::Custom: {
518 case TargetLowering::Promote: {
521 "Can only promote stores to same size type");
524 ST->getOriginalAlign(), MMOFlags, AAInfo);
534 EVT StVT =
ST->getMemoryVT();
537 auto &
DL = DAG.getDataLayout();
539 if (StWidth != StSize) {
544 Value = DAG.getZeroExtendInReg(
Value, dl, StVT);
546 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), NVT,
547 ST->getOriginalAlign(), MMOFlags, AAInfo);
553 unsigned LogStWidth =
Log2_32(StWidthBits);
555 unsigned RoundWidth = 1 << LogStWidth;
556 assert(RoundWidth < StWidthBits);
557 unsigned ExtraWidth = StWidthBits - RoundWidth;
558 assert(ExtraWidth < RoundWidth);
559 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
560 "Store size not an integral number of bytes!");
564 unsigned IncrementSize;
566 if (
DL.isLittleEndian()) {
569 Lo = DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(),
570 RoundVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
573 IncrementSize = RoundWidth / 8;
578 DAG.getConstant(RoundWidth, dl,
579 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
580 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
581 ST->getPointerInfo().getWithOffset(IncrementSize),
582 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
589 DAG.getConstant(ExtraWidth, dl,
590 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
591 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
ST->getPointerInfo(), RoundVT,
592 ST->getOriginalAlign(), MMOFlags, AAInfo);
595 IncrementSize = RoundWidth / 8;
597 DAG.getConstant(IncrementSize, dl,
598 Ptr.getValueType()));
600 ST->getPointerInfo().getWithOffset(IncrementSize),
601 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
608 switch (TLI.getTruncStoreAction(
ST->getValue().getValueType(), StVT)) {
610 case TargetLowering::Legal: {
611 EVT MemVT =
ST->getMemoryVT();
614 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
615 *
ST->getMemOperand())) {
617 ReplaceNode(
SDValue(ST, 0), Result);
621 case TargetLowering::Custom: {
627 case TargetLowering::Expand:
629 "Vector Stores are handled in LegalizeVectorOps");
634 if (TLI.isTypeLegal(StVT)) {
637 ST->getOriginalAlign(), MMOFlags, AAInfo);
642 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
645 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), StVT,
646 ST->getOriginalAlign(), MMOFlags, AAInfo);
655void SelectionDAGLegalize::LegalizeLoadOps(
SDNode *
Node) {
664 LLVM_DEBUG(
dbgs() <<
"Legalizing non-extending load operation\n");
665 MVT VT =
Node->getSimpleValueType(0);
669 switch (TLI.getOperationAction(
Node->getOpcode(), VT)) {
671 case TargetLowering::Legal: {
672 EVT MemVT =
LD->getMemoryVT();
676 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
677 *
LD->getMemOperand())) {
678 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
682 case TargetLowering::Custom:
683 if (
SDValue Res = TLI.LowerOperation(RVal, DAG)) {
689 case TargetLowering::Promote: {
690 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
692 "Can only promote loads to same size type");
694 SDValue Res = DAG.getLoad(NVT, dl, Chain,
Ptr,
LD->getMemOperand());
702 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), RVal);
703 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), RChain);
705 UpdatedNodes->insert(RVal.
getNode());
706 UpdatedNodes->insert(RChain.
getNode());
714 EVT SrcVT =
LD->getMemoryVT();
728 TLI.getLoadExtAction(ExtType,
Node->getValueType(0), MVT::i1) ==
729 TargetLowering::Promote)) {
743 Chain,
Ptr,
LD->getPointerInfo(), NVT,
744 LD->getOriginalAlign(), MMOFlags, AAInfo);
752 Result, DAG.getValueType(SrcVT));
756 Result.getValueType(), Result,
757 DAG.getValueType(SrcVT));
765 unsigned LogSrcWidth =
Log2_32(SrcWidthBits);
767 unsigned RoundWidth = 1 << LogSrcWidth;
768 assert(RoundWidth < SrcWidthBits);
769 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
770 assert(ExtraWidth < RoundWidth);
771 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
772 "Load size not an integral number of bytes!");
776 unsigned IncrementSize;
777 auto &
DL = DAG.getDataLayout();
779 if (
DL.isLittleEndian()) {
783 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
787 IncrementSize = RoundWidth / 8;
790 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
791 LD->getPointerInfo().getWithOffset(IncrementSize),
792 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
802 DAG.getConstant(RoundWidth, dl,
803 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
811 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
812 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
816 IncrementSize = RoundWidth / 8;
820 LD->getPointerInfo().getWithOffset(IncrementSize),
821 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
831 DAG.getConstant(ExtraWidth, dl,
832 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
840 bool isCustom =
false;
841 switch (TLI.getLoadExtAction(ExtType,
Node->getValueType(0),
844 case TargetLowering::Custom:
847 case TargetLowering::Legal:
859 EVT MemVT =
LD->getMemoryVT();
861 if (!TLI.allowsMemoryAccess(*DAG.getContext(),
DL, MemVT,
862 *
LD->getMemOperand())) {
863 std::tie(
Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
868 case TargetLowering::Expand: {
869 EVT DestVT =
Node->getValueType(0);
875 (TLI.isTypeLegal(SrcVT) ||
876 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
883 SrcVT,
LD->getMemOperand());
886 Value = DAG.getNode(ExtendOp, dl,
Node->getValueType(0), Load);
887 Chain =
Load.getValue(1);
896 if (SVT == MVT::f16 || SVT == MVT::bf16) {
902 Ptr, ISrcVT,
LD->getMemOperand());
906 Chain =
Result.getValue(1);
912 "Vector Loads are handled in LegalizeVectorOps");
919 "EXTLOAD should always be supported!");
923 Node->getValueType(0),
925 LD->getMemOperand());
930 Result, DAG.getValueType(SrcVT));
932 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
934 Chain =
Result.getValue(1);
945 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
947 UpdatedNodes->insert(
Value.getNode());
948 UpdatedNodes->insert(Chain.
getNode());
955void SelectionDAGLegalize::LegalizeOp(
SDNode *
Node) {
964 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
965 assert(TLI.getTypeAction(*DAG.getContext(),
Node->getValueType(i)) ==
966 TargetLowering::TypeLegal &&
967 "Unexpected illegal type!");
970 assert((TLI.getTypeAction(*DAG.getContext(),
Op.getValueType()) ==
971 TargetLowering::TypeLegal ||
974 "Unexpected illegal type!");
979 bool SimpleFinishLegalizing =
true;
980 switch (
Node->getOpcode()) {
985 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
988 Action = TLI.getOperationAction(
Node->getOpcode(),
989 Node->getValueType(0));
992 Action = TLI.getOperationAction(
Node->getOpcode(),
993 Node->getValueType(0));
994 if (Action != TargetLowering::Promote)
995 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
999 Action = TLI.getOperationAction(
Node->getOpcode(),
1000 Node->getOperand(1).getValueType());
1011 Action = TLI.getOperationAction(
Node->getOpcode(),
1012 Node->getOperand(0).getValueType());
1025 Action = TLI.getOperationAction(
Node->getOpcode(),
1026 Node->getOperand(1).getValueType());
1029 EVT InnerType = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1030 Action = TLI.getOperationAction(
Node->getOpcode(), InnerType);
1034 Action = TLI.getOperationAction(
Node->getOpcode(),
1035 Node->getOperand(1).getValueType());
1044 unsigned Opc =
Node->getOpcode();
1049 : (Opc ==
ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1051 unsigned CompareOperand = Opc ==
ISD::BR_CC ? 2
1055 MVT OpVT =
Node->getOperand(CompareOperand).getSimpleValueType();
1057 cast<CondCodeSDNode>(
Node->getOperand(CCOperand))->get();
1058 Action = TLI.getCondCodeAction(CCCode, OpVT);
1059 if (Action == TargetLowering::Legal) {
1061 Action = TLI.getOperationAction(
Node->getOpcode(),
1062 Node->getValueType(0));
1064 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
1072 SimpleFinishLegalizing =
false;
1079 SimpleFinishLegalizing =
false;
1092 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1093 if (Action == TargetLowering::Legal)
1094 Action = TargetLowering::Expand;
1104 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1105 if (Action == TargetLowering::Legal)
1106 Action = TargetLowering::Custom;
1111 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1117 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::i64);
1124 Action = TargetLowering::Legal;
1127 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1128 if (Action == TargetLowering::Expand) {
1132 Node->getOperand(0));
1139 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1140 if (Action == TargetLowering::Expand) {
1144 Node->getOperand(0));
1160 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1170 unsigned Scale =
Node->getConstantOperandVal(2);
1171 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
1172 Node->getValueType(0), Scale);
1176 Action = TLI.getOperationAction(
Node->getOpcode(),
1177 cast<MaskedScatterSDNode>(
Node)->getValue().getValueType());
1180 Action = TLI.getOperationAction(
Node->getOpcode(),
1181 cast<MaskedStoreSDNode>(
Node)->getValue().getValueType());
1183 case ISD::VP_SCATTER:
1184 Action = TLI.getOperationAction(
1186 cast<VPScatterSDNode>(
Node)->getValue().getValueType());
1189 Action = TLI.getOperationAction(
1191 cast<VPStoreSDNode>(
Node)->getValue().getValueType());
1193 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1194 Action = TLI.getOperationAction(
1196 cast<VPStridedStoreSDNode>(
Node)->getValue().getValueType());
1214 Action = TLI.getOperationAction(
1215 Node->getOpcode(),
Node->getOperand(0).getValueType());
1219 case ISD::VP_REDUCE_FADD:
1220 case ISD::VP_REDUCE_FMUL:
1221 case ISD::VP_REDUCE_ADD:
1222 case ISD::VP_REDUCE_MUL:
1223 case ISD::VP_REDUCE_AND:
1224 case ISD::VP_REDUCE_OR:
1225 case ISD::VP_REDUCE_XOR:
1226 case ISD::VP_REDUCE_SMAX:
1227 case ISD::VP_REDUCE_SMIN:
1228 case ISD::VP_REDUCE_UMAX:
1229 case ISD::VP_REDUCE_UMIN:
1230 case ISD::VP_REDUCE_FMAX:
1231 case ISD::VP_REDUCE_FMIN:
1232 case ISD::VP_REDUCE_FMAXIMUM:
1233 case ISD::VP_REDUCE_FMINIMUM:
1234 case ISD::VP_REDUCE_SEQ_FADD:
1235 case ISD::VP_REDUCE_SEQ_FMUL:
1236 Action = TLI.getOperationAction(
1237 Node->getOpcode(),
Node->getOperand(1).getValueType());
1239 case ISD::VP_CTTZ_ELTS:
1240 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
1241 Action = TLI.getOperationAction(
Node->getOpcode(),
1242 Node->getOperand(0).getValueType());
1246 Action = TLI.getCustomOperationAction(*
Node);
1248 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1253 if (SimpleFinishLegalizing) {
1255 switch (
Node->getOpcode()) {
1274 NewNode = DAG.UpdateNodeOperands(
Node, Op0, SAO);
1294 NewNode = DAG.UpdateNodeOperands(
Node, Op0, Op1, SAO);
1300 if (NewNode !=
Node) {
1301 ReplaceNode(
Node, NewNode);
1305 case TargetLowering::Legal:
1308 case TargetLowering::Custom:
1316 if (
Node->getNumValues() == 1) {
1320 Node->getValueType(0) == MVT::Glue) &&
1321 "Type mismatch for custom legalized operation");
1329 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i) {
1333 Node->getValueType(i) == MVT::Glue) &&
1334 "Type mismatch for custom legalized operation");
1338 ReplaceNode(
Node, ResultVals.
data());
1343 case TargetLowering::Expand:
1344 if (ExpandNode(
Node))
1347 case TargetLowering::LibCall:
1348 ConvertNodeToLibcall(
Node);
1350 case TargetLowering::Promote:
1356 switch (
Node->getOpcode()) {
1369 return LegalizeLoadOps(
Node);
1371 return LegalizeStoreOps(
Node);
1375SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(
SDValue Op) {
1395 if (
ST->isIndexed() ||
ST->isTruncatingStore() ||
1396 ST->getValue() != Vec)
1401 if (!
ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1410 ST->hasPredecessor(
Op.getNode()))
1423 StackPtr = DAG.CreateStackTemporary(VecVT);
1426 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO);
1430 Align ElementAlignment =
1431 std::min(cast<StoreSDNode>(Ch)->
getAlign(),
1432 DAG.getDataLayout().getPrefTypeAlign(
1433 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1435 if (
Op.getValueType().isVector()) {
1436 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1437 Op.getValueType(),
Idx);
1438 NewLoad = DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr,
1441 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1442 NewLoad = DAG.getExtLoad(
ISD::EXTLOAD, dl,
Op.getValueType(), Ch, StackPtr,
1448 DAG.ReplaceAllUsesOfValueWith(Ch,
SDValue(NewLoad.
getNode(), 1));
1454 NewLoadOperands[0] = Ch;
1456 SDValue(DAG.UpdateNodeOperands(NewLoad.
getNode(), NewLoadOperands), 0);
1460SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(
SDValue Op) {
1461 assert(
Op.getValueType().isVector() &&
"Non-vector insert subvector!");
1472 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1477 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1480 Idx = DAG.getFreeze(
Idx);
1485 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, PartVT,
Idx);
1489 Ch, dl, Part, SubStackPtr,
1493 TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1496 Ch = DAG.getTruncStore(
1497 Ch, dl, Part, SubStackPtr,
1503 return DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1509 "Unexpected opcode!");
1515 EVT VT =
Node->getValueType(0);
1517 :
Node->getOperand(0).getValueType();
1519 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1520 int FI = cast<FrameIndexSDNode>(FIPtr.
getNode())->getIndex();
1527 assert(TypeByteSize > 0 &&
"Vector element type too small for stack store!");
1531 bool Truncate = isa<BuildVectorSDNode>(
Node) &&
1532 MemVT.
bitsLT(
Node->getOperand(0).getValueType());
1535 for (
unsigned i = 0, e =
Node->getNumOperands(); i != e; ++i) {
1537 if (
Node->getOperand(i).isUndef())
continue;
1539 unsigned Offset = TypeByteSize*i;
1545 Stores.
push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1549 Stores.
push_back(DAG.getStore(DAG.getEntryNode(), dl,
Node->getOperand(i),
1554 if (!Stores.
empty())
1557 StoreChain = DAG.getEntryNode();
1560 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1566void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1569 EVT FloatVT =
Value.getValueType();
1571 State.FloatVT = FloatVT;
1574 if (TLI.isTypeLegal(IVT)) {
1577 State.SignBit = NumBits - 1;
1583 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1586 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1591 State.Chain = DAG.getStore(DAG.getEntryNode(),
DL,
Value, State.FloatPtr,
1592 State.FloatPointerInfo);
1599 State.IntPointerInfo = State.FloatPointerInfo;
1602 unsigned ByteOffset = (NumBits / 8) - 1;
1609 State.IntPtr = IntPtr;
1610 State.IntValue = DAG.getExtLoad(
ISD::EXTLOAD,
DL, LoadTy, State.Chain, IntPtr,
1611 State.IntPointerInfo, MVT::i8);
1618SDValue SelectionDAGLegalize::modifySignAsInt(
const FloatSignAsInt &State,
1625 SDValue Chain = DAG.getTruncStore(State.Chain,
DL, NewIntValue, State.IntPtr,
1626 State.IntPointerInfo, MVT::i8);
1627 return DAG.getLoad(State.FloatVT,
DL, Chain, State.FloatPtr,
1628 State.FloatPointerInfo);
1637 FloatSignAsInt SignAsInt;
1638 getSignAsIntValue(SignAsInt,
DL, Sign);
1640 EVT IntVT = SignAsInt.IntValue.getValueType();
1641 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1647 if (TLI.isOperationLegalOrCustom(
ISD::FABS, FloatVT) &&
1648 TLI.isOperationLegalOrCustom(
ISD::FNEG, FloatVT)) {
1651 SDValue Cond = DAG.getSetCC(
DL, getSetCCResultType(IntVT), SignBit,
1653 return DAG.getSelect(
DL, FloatVT,
Cond, NegValue, AbsValue);
1657 FloatSignAsInt MagAsInt;
1658 getSignAsIntValue(MagAsInt,
DL, Mag);
1659 EVT MagVT = MagAsInt.IntValue.getValueType();
1660 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask,
DL, MagVT);
1665 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1666 EVT ShiftVT = IntVT;
1672 if (ShiftAmount > 0) {
1673 SDValue ShiftCnst = DAG.getConstant(ShiftAmount,
DL, ShiftVT);
1675 }
else if (ShiftAmount < 0) {
1676 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount,
DL, ShiftVT);
1685 Flags.setDisjoint(
true);
1691 return modifySignAsInt(MagAsInt,
DL, CopiedSign);
1697 FloatSignAsInt SignAsInt;
1698 getSignAsIntValue(SignAsInt,
DL,
Node->getOperand(0));
1699 EVT IntVT = SignAsInt.IntValue.getValueType();
1702 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1707 return modifySignAsInt(SignAsInt,
DL, SignFlip);
1715 EVT FloatVT =
Value.getValueType();
1722 FloatSignAsInt ValueAsInt;
1723 getSignAsIntValue(ValueAsInt,
DL,
Value);
1724 EVT IntVT = ValueAsInt.IntValue.getValueType();
1725 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask,
DL, IntVT);
1728 return modifySignAsInt(ValueAsInt,
DL, ClearedSign);
1731void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(
SDNode*
Node,
1733 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1734 assert(SPReg &&
"Target cannot require DYNAMIC_STACKALLOC expansion and"
1735 " not tell us which reg is the stack pointer!");
1737 EVT VT =
Node->getValueType(0);
1745 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1748 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1750 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1758 if (Alignment > StackAlign)
1760 DAG.getConstant(-Alignment.
value(), dl, VT));
1761 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);
1763 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0,
SDValue(), dl);
1775 return EmitStackConvert(
SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1783 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1786 if ((SrcVT.
bitsGT(SlotVT) &&
1787 !TLI.isTruncStoreLegalOrCustom(
SrcOp.getValueType(), SlotVT)) ||
1788 (SlotVT.
bitsLT(DestVT) &&
1789 !TLI.isLoadExtLegalOrCustom(
ISD::EXTLOAD, DestVT, SlotVT)))
1793 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1794 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1806 if (SrcVT.
bitsGT(SlotVT))
1807 Store = DAG.getTruncStore(Chain, dl,
SrcOp, FIPtr, PtrInfo,
1811 Store = DAG.getStore(Chain, dl,
SrcOp, FIPtr, PtrInfo, SrcAlign);
1815 if (SlotVT.
bitsEq(DestVT))
1816 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1819 return DAG.getExtLoad(
ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1832 SDValue Ch = DAG.getTruncStore(
1833 DAG.getEntryNode(), dl,
Node->getOperand(0), StackPtr,
1835 Node->getValueType(0).getVectorElementType());
1837 Node->getValueType(0), dl, Ch, StackPtr,
1844 unsigned NumElems =
Node->getNumOperands();
1846 EVT VT =
Node->getValueType(0);
1858 for (
unsigned i = 0; i < NumElems; ++i) {
1869 while (IntermedVals.
size() > 2) {
1870 NewIntermedVals.
clear();
1871 for (
unsigned i = 0, e = (IntermedVals.
size() & ~1u); i < e; i += 2) {
1877 FinalIndices.
reserve(IntermedVals[i].second.
size() +
1878 IntermedVals[i+1].second.
size());
1881 for (
unsigned j = 0, f = IntermedVals[i].second.
size(); j != f;
1884 FinalIndices.
push_back(IntermedVals[i].second[j]);
1886 for (
unsigned j = 0, f = IntermedVals[i+1].second.
size(); j != f;
1888 ShuffleVec[k] = NumElems + j;
1889 FinalIndices.
push_back(IntermedVals[i+1].second[j]);
1895 IntermedVals[i+1].first,
1900 std::make_pair(Shuffle, std::move(FinalIndices)));
1905 if ((IntermedVals.
size() & 1) != 0)
1908 IntermedVals.
swap(NewIntermedVals);
1912 "Invalid number of intermediate vectors");
1913 SDValue Vec1 = IntermedVals[0].first;
1915 if (IntermedVals.
size() > 1)
1916 Vec2 = IntermedVals[1].first;
1921 for (
unsigned i = 0, e = IntermedVals[0].second.
size(); i != e; ++i)
1922 ShuffleVec[IntermedVals[0].second[i]] = i;
1923 for (
unsigned i = 0, e = IntermedVals[1].second.
size(); i != e; ++i)
1924 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1938 unsigned NumElems =
Node->getNumOperands();
1941 EVT VT =
Node->getValueType(0);
1942 EVT OpVT =
Node->getOperand(0).getValueType();
1947 bool isOnlyLowElement =
true;
1948 bool MoreThanTwoValues =
false;
1950 for (
unsigned i = 0; i < NumElems; ++i) {
1955 isOnlyLowElement =
false;
1956 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1961 }
else if (!Value2.
getNode()) {
1964 }
else if (V != Value1 && V != Value2) {
1965 MoreThanTwoValues =
true;
1970 return DAG.getUNDEF(VT);
1972 if (isOnlyLowElement)
1978 for (
unsigned i = 0, e = NumElems; i !=
e; ++i) {
1980 dyn_cast<ConstantFPSDNode>(
Node->getOperand(i))) {
1983 dyn_cast<ConstantSDNode>(
Node->getOperand(i))) {
2002 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
2003 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2005 VT, dl, DAG.getEntryNode(), CPIdx,
2011 for (
unsigned i = 0; i < NumElems; ++i) {
2012 if (
Node->getOperand(i).isUndef())
2017 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.
size())) {
2018 if (!MoreThanTwoValues) {
2020 for (
unsigned i = 0; i < NumElems; ++i) {
2024 ShuffleVec[i] =
V == Value1 ? 0 : NumElems;
2026 if (TLI.isShuffleMaskLegal(ShuffleVec,
Node->getValueType(0))) {
2033 Vec2 = DAG.getUNDEF(VT);
2036 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2046 return ExpandVectorBuildThroughStack(
Node);
2051 EVT VT =
Node->getValueType(0);
2054 return DAG.getSplatBuildVector(VT,
DL, SplatVal);
2065 EVT CodePtrTy = TLI.getPointerTy(DAG.getDataLayout());
2067 if (
const char *LibcallName = TLI.getLibcallName(LC))
2068 Callee = DAG.getExternalSymbol(LibcallName, CodePtrTy);
2070 Callee = DAG.getUNDEF(CodePtrTy);
2071 DAG.getContext()->emitError(
Twine(
"no libcall available for ") +
2072 Node->getOperationName(&DAG));
2075 EVT RetVT =
Node->getValueType(0);
2082 SDValue InChain = DAG.getEntryNode();
2087 const Function &
F = DAG.getMachineFunction().getFunction();
2089 TLI.isInTailCallPosition(DAG,
Node, TCChain) &&
2090 (
RetTy ==
F.getReturnType() ||
F.getReturnType()->isVoidTy());
2095 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT,
isSigned);
2098 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2100 .setTailCall(isTailCall)
2101 .setSExtResult(signExtend)
2102 .setZExtResult(!signExtend)
2103 .setIsPostTypeLegalization(
true);
2105 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2110 return {DAG.getRoot(), DAG.getRoot()};
2122 EVT ArgVT =
Op.getValueType();
2126 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT,
isSigned);
2128 Args.push_back(Entry);
2131 return ExpandLibCall(LC,
Node, std::move(Args),
isSigned);
2134void SelectionDAGLegalize::ExpandFrexpLibCall(
2137 EVT VT =
Node->getValueType(0);
2138 EVT ExpVT =
Node->getValueType(1);
2146 FPArgEntry.
Node = FPOp;
2147 FPArgEntry.
Ty = ArgTy;
2149 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2151 PtrArgEntry.
Node = StackSlot;
2152 PtrArgEntry.
Ty = PointerType::get(*DAG.getContext(),
2153 DAG.getDataLayout().getAllocaAddrSpace());
2158 auto [
Call, Chain] = ExpandLibCall(LC,
Node, std::move(Args),
false);
2162 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2166 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2168 LoadExp.
getValue(1), DAG.getRoot());
2169 DAG.setRoot(OutputChain);
2175void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2178 if (LC == RTLIB::UNKNOWN_LIBCALL)
2181 if (
Node->isStrictFPOpcode()) {
2182 EVT RetVT =
Node->getValueType(0);
2186 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2189 Node->getOperand(0));
2191 Results.push_back(Tmp.second);
2193 SDValue Tmp = ExpandLibCall(LC,
Node,
false).first;
2199void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2207 Call_F32, Call_F64, Call_F80,
2208 Call_F128, Call_PPCF128);
2219 switch (
Node->getSimpleValueType(0).SimpleTy) {
2221 case MVT::i8: LC = Call_I8;
break;
2222 case MVT::i16: LC = Call_I16;
break;
2223 case MVT::i32: LC = Call_I32;
break;
2224 case MVT::i64: LC = Call_I64;
break;
2225 case MVT::i128: LC = Call_I128;
break;
2232void SelectionDAGLegalize::ExpandArgFPLibCall(
SDNode*
Node,
2239 EVT InVT =
Node->getOperand(
Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2241 Call_F32, Call_F64, Call_F80,
2242 Call_F128, Call_PPCF128);
2248SelectionDAGLegalize::ExpandDivRemLibCall(
SDNode *
Node,
2250 unsigned Opcode =
Node->getOpcode();
2254 switch (
Node->getSimpleValueType(0).SimpleTy) {
2256 case MVT::i8: LC=
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
2257 case MVT::i16: LC=
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
2258 case MVT::i32: LC=
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
2259 case MVT::i64: LC=
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
2260 case MVT::i128: LC=
isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
2266 SDValue InChain = DAG.getEntryNode();
2268 EVT RetVT =
Node->getValueType(0);
2274 EVT ArgVT =
Op.getValueType();
2280 Args.push_back(Entry);
2284 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2286 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2289 Args.push_back(Entry);
2291 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2292 TLI.getPointerTy(DAG.getDataLayout()));
2298 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2303 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2315 switch (
Node->getSimpleValueType(0).SimpleTy) {
2317 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2318 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2319 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2320 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2321 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2344SelectionDAGLegalize::ExpandSinCosLibCall(
SDNode *
Node,
2347 switch (
Node->getSimpleValueType(0).SimpleTy) {
2349 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2350 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2351 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2352 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2353 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2359 SDValue InChain = DAG.getEntryNode();
2361 EVT RetVT =
Node->getValueType(0);
2370 Entry.IsSExt =
false;
2371 Entry.IsZExt =
false;
2372 Args.push_back(Entry);
2375 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2376 Entry.Node = SinPtr;
2377 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2378 Entry.IsSExt =
false;
2379 Entry.IsZExt =
false;
2380 Args.push_back(Entry);
2383 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2384 Entry.Node = CosPtr;
2385 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2386 Entry.IsSExt =
false;
2387 Entry.IsZExt =
false;
2388 Args.push_back(Entry);
2390 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2391 TLI.getPointerTy(DAG.getDataLayout()));
2395 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2396 TLI.getLibcallCallingConv(LC),
Type::getVoidTy(*DAG.getContext()), Callee,
2399 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2409 EVT VT =
Node->getValueType(0);
2412 EVT ExpVT =
N.getValueType();
2414 if (AsIntVT ==
EVT())
2427 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2432 const int Precision = APFloat::semanticsPrecision(FltSem);
2434 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2435 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2437 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2439 const APFloat One(FltSem,
"1.0");
2440 APFloat ScaleUpK =
scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2444 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2453 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2459 DAG.getSetCC(dl, SetCCVT,
N, DoubleMaxExp,
ISD::SETUGT);
2461 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2473 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2474 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2479 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2484 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2488 SDValue ScaleDownTwice = DAG.getSetCC(
2489 dl, SetCCVT,
N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2501 DAG.getNode(
ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small,
X));
2505 DAG.getNode(
ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small,
N));
2510 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2511 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2514 ExponentShiftAmt, NUW_NSW);
2516 return DAG.getNode(
ISD::FMUL, dl, VT, NewX, AsFP);
2523 EVT ExpVT =
Node->getValueType(1);
2525 if (AsIntVT ==
EVT())
2530 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2547 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2551 SDValue SmallestNormalizedInt = DAG.getConstant(
2557 DAG.getConstant(
APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2563 FractSignMaskVal.
setBit(BitSize - 1);
2566 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2568 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2570 const APFloat One(FltSem,
"1.0");
2574 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2576 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2580 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2586 SDValue AddNegSmallestNormal =
2588 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2592 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt,
ISD::SETULT);
2594 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2609 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2612 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2615 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2622 SDValue FPHalf = DAG.getConstant(
Half.bitcastToAPInt(), dl, AsIntVT);
2635 return DAG.getMergeValues({Result0, Result1}, dl);
2646 EVT DestVT =
Node->getValueType(0);
2648 unsigned OpNo =
Node->isStrictFPOpcode() ? 1 : 0;
2654 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2655 (DestVT.
bitsLE(MVT::f64) ||
2659 LLVM_DEBUG(
dbgs() <<
"32-bit [signed|unsigned] integer to float/double "
2663 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2670 DAG.getConstant(0x80000000u, dl, MVT::i32));
2673 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2676 if (DAG.getDataLayout().isBigEndian())
2679 SDValue MemChain = DAG.getEntryNode();
2682 SDValue Store1 = DAG.getStore(MemChain, dl,
Lo, StackSlot,
2695 SDValue Bias = DAG.getConstantFP(
2696 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2697 : llvm::bit_cast<double>(0x4330000000000000ULL),
2702 if (
Node->isStrictFPOpcode()) {
2704 {
Node->getOperand(0), Load, Bias});
2707 std::pair<SDValue, SDValue> ResultPair;
2709 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2710 Result = ResultPair.first;
2711 Chain = ResultPair.second;
2717 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2726 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2727 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2728 LLVM_DEBUG(
dbgs() <<
"Converting unsigned i32/i64 to f32/f64\n");
2743 EVT SetCCVT = getSetCCResultType(SrcVT);
2745 SDValue SignBitTest = DAG.getSetCC(
2746 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT),
ISD::SETLT);
2748 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2749 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2751 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2756 if (
Node->isStrictFPOpcode()) {
2759 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest,
Or, Op0);
2761 {
Node->getOperand(0), InCvt });
2769 Flags.setNoFPExcept(
Node->getFlags().hasNoFPExcept());
2770 Fast->setFlags(Flags);
2771 Flags.setNoFPExcept(
true);
2779 return DAG.getSelect(dl, DestVT, SignBitTest, Slow,
Fast);
2783 if (!TLI.isOperationLegalOrCustom(
2790 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2792 "Cannot perform lossless SINT_TO_FP!");
2795 if (
Node->isStrictFPOpcode()) {
2797 {
Node->getOperand(0), Op0 });
2801 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2804 Four = DAG.getIntPtrConstant(4, dl);
2805 SDValue CstOffset = DAG.getSelect(dl,
Zero.getValueType(),
2806 SignSet, Four, Zero);
2815 case MVT::i8 : FF = 0x43800000ULL;
break;
2816 case MVT::i16: FF = 0x47800000ULL;
break;
2817 case MVT::i32: FF = 0x4F800000ULL;
break;
2818 case MVT::i64: FF = 0x5F800000ULL;
break;
2820 if (DAG.getDataLayout().isLittleEndian())
2822 Constant *FudgeFactor = ConstantInt::get(
2826 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2827 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2831 if (DestVT == MVT::f32)
2832 FudgeInReg = DAG.getLoad(
2833 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2842 LegalizeOp(
Load.getNode());
2846 if (
Node->isStrictFPOpcode()) {
2848 { Tmp1.
getValue(1), Tmp1, FudgeInReg });
2849 Chain =
Result.getValue(1);
2853 return DAG.getNode(
ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2861void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2863 bool IsStrict =
N->isStrictFPOpcode();
2866 EVT DestVT =
N->getValueType(0);
2867 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2874 unsigned OpToUse = 0;
2882 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2890 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2902 DAG.
getNode(OpToUse, dl, {DestVT, MVT::Other},
2905 dl, NewInTy, LegalOp)});
2912 DAG.getNode(OpToUse, dl, DestVT,
2914 dl, NewInTy, LegalOp)));
2922void SelectionDAGLegalize::PromoteLegalFP_TO_INT(
SDNode *
N,
const SDLoc &dl,
2924 bool IsStrict =
N->isStrictFPOpcode();
2927 EVT DestVT =
N->getValueType(0);
2928 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2930 EVT NewOutTy = DestVT;
2932 unsigned OpToUse = 0;
2942 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2947 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2956 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2957 Operation = DAG.getNode(OpToUse, dl, VTs,
N->getOperand(0), LegalOp);
2959 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2974 unsigned Opcode =
Node->getOpcode();
2977 EVT NewOutTy =
Node->getValueType(0);
2982 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2989 Node->getOperand(1));
2995 EVT VT =
Op.getValueType();
2996 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3001 if (TLI.isOperationLegalOrPromote(
ISD::CTPOP, VT)) {
3007 DAG.getConstant(1ULL << (--i), dl, ShVT));
3012 return DAG.getNode(
ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
3016 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
3017 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
3018 MVT ScalarVT =
Node->getSimpleValueType(0);
3026 assert(
Node->getOperand(0).getValueType().isFloatingPoint() &&
3027 "Only FP promotion is supported");
3031 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
3032 if (
Node->getOperand(j).getValueType().isVector() &&
3037 assert(
Node->getOperand(j).getValueType().isFloatingPoint() &&
3038 "Only FP promotion is supported");
3050 DAG.getIntPtrConstant(0,
DL,
true));
3053bool SelectionDAGLegalize::ExpandNode(
SDNode *
Node) {
3057 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3059 switch (
Node->getOpcode()) {
3061 if ((Tmp1 = TLI.expandABS(
Node, DAG)))
3066 if ((Tmp1 = TLI.expandABD(
Node, DAG)))
3073 if ((Tmp1 = TLI.expandAVG(
Node, DAG)))
3077 if ((Tmp1 = TLI.expandCTPOP(
Node, DAG)))
3082 if ((Tmp1 = TLI.expandCTLZ(
Node, DAG)))
3087 if ((Tmp1 = TLI.expandCTTZ(
Node, DAG)))
3091 if ((Tmp1 = TLI.expandBITREVERSE(
Node, DAG)))
3095 if ((Tmp1 = TLI.expandBSWAP(
Node, DAG)))
3099 Results.push_back(ExpandPARITY(
Node->getOperand(0), dl));
3104 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3107 SDValue CfaArg = DAG.getSExtOrTrunc(
Node->getOperand(0), dl,
3108 TLI.getPointerTy(DAG.getDataLayout()));
3116 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3122 Results.push_back(DAG.getConstant(1, dl,
Node->getValueType(0)));
3139 DAG.getConstant(0, dl,
Node->getValueType(0)));
3145 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3151 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3152 SDValue Swap = DAG.getAtomicCmpSwap(
3154 Node->getOperand(0),
Node->getOperand(1), Zero, Zero,
3155 cast<AtomicSDNode>(
Node)->getMemOperand());
3164 Node->getOperand(0),
Node->getOperand(2),
Node->getOperand(1),
3165 cast<AtomicSDNode>(
Node)->getMemOperand());
3173 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3174 SDValue Res = DAG.getAtomicCmpSwap(
3176 Node->getOperand(0),
Node->getOperand(1),
Node->getOperand(2),
3177 Node->getOperand(3), cast<MemSDNode>(
Node)->getMemOperand());
3183 EVT AtomicType = cast<AtomicSDNode>(
Node)->getMemoryVT();
3184 EVT OuterType =
Node->getValueType(0);
3185 switch (TLI.getExtendForAtomicOps()) {
3188 DAG.getValueType(AtomicType));
3190 Node->getOperand(2), DAG.getValueType(AtomicType));
3195 DAG.getValueType(AtomicType));
3196 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3200 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3201 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3217 EVT VT =
Node->getValueType(0);
3221 cast<VTSDNode>(
RHS->getOperand(1))->getVT() == AN->
getMemoryVT())
3222 RHS =
RHS->getOperand(0);
3226 Node->getOperand(0),
Node->getOperand(1),
3236 for (
unsigned i = 0; i <
Node->getNumValues(); i++)
3240 EVT VT =
Node->getValueType(0);
3242 Results.push_back(DAG.getConstant(0, dl, VT));
3245 Results.push_back(DAG.getConstantFP(0, dl, VT));
3252 if (TLI.isStrictFPEnabled())
3256 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3257 Node->getValueType(0))
3258 == TargetLowering::Legal)
3262 if ((Tmp1 = EmitStackConvert(
Node->getOperand(1),
Node->getValueType(0),
3263 Node->getValueType(0), dl,
3264 Node->getOperand(0)))) {
3266 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_ROUND node\n");
3271 if ((Tmp1 = TLI.expandFP_ROUND(
Node, DAG))) {
3279 if ((Tmp1 = EmitStackConvert(
Node->getOperand(0),
Node->getValueType(0),
3280 Node->getValueType(0), dl)))
3286 if (TLI.isStrictFPEnabled())
3290 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3291 Node->getValueType(0))
3292 == TargetLowering::Legal)
3296 if ((Tmp1 = EmitStackConvert(
3297 Node->getOperand(1),
Node->getOperand(1).getValueType(),
3298 Node->getValueType(0), dl,
Node->getOperand(0)))) {
3300 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_EXTEND node\n");
3306 EVT SrcVT =
Op.getValueType();
3307 EVT DstVT =
Node->getValueType(0);
3313 if ((Tmp1 = EmitStackConvert(
Op, SrcVT, DstVT, dl)))
3323 if (
Op.getValueType() == MVT::bf16) {
3327 Op = DAG.getAnyExtOrTrunc(
Op, dl, MVT::i32);
3331 DAG.getConstant(16, dl,
3332 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3335 if (
Node->getValueType(0) != MVT::f32)
3342 if (
Op.getValueType() != MVT::f32)
3344 DAG.getIntPtrConstant(0, dl,
true));
3346 if (!DAG.isKnownNeverSNaN(
Op)) {
3351 DAG.getConstant(16, dl,
3352 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3355 if (
Node->getValueType(0) == MVT::bf16) {
3359 Op = DAG.getAnyExtOrTrunc(
Op, dl,
Node->getValueType(0));
3365 EVT ExtraVT = cast<VTSDNode>(
Node->getOperand(1))->getVT();
3366 EVT VT =
Node->getValueType(0);
3376 SDValue One = DAG.getConstant(1, dl, VT);
3386 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3389 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3391 Node->getOperand(0), ShiftCst);
3398 if (TLI.expandUINT_TO_FP(
Node, Tmp1, Tmp2, DAG)) {
3400 if (
Node->isStrictFPOpcode())
3407 if ((Tmp1 = ExpandLegalINT_TO_FP(
Node, Tmp2))) {
3409 if (
Node->isStrictFPOpcode())
3414 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG))
3418 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG)) {
3420 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_SINT node\n");
3425 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG))
3429 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG)) {
3431 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node,1), Tmp2);
3434 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_UINT node\n");
3440 Results.push_back(TLI.expandFP_TO_INT_SAT(
Node, DAG));
3450 if (
Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3453 Node->getOperand(0));
3455 Tmp1 = ExpandExtractFromVectorThroughStack(
SDValue(
Node, 0));
3465 Results.push_back(ExpandVectorBuildThroughStack(
Node));
3477 EVT VT =
Node->getValueType(0);
3481 if (!TLI.isTypeLegal(EltVT)) {
3482 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3487 if (NewEltVT.
bitsLT(EltVT)) {
3503 unsigned int factor =
3511 for (
unsigned fi = 0; fi < factor; ++fi)
3515 for (
unsigned fi = 0; fi < factor; ++fi)
3526 for (
unsigned i = 0; i != NumElems; ++i) {
3534 DAG.getVectorIdxConstant(
Idx, dl)));
3538 DAG.getVectorIdxConstant(
Idx - NumElems, dl)));
3541 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3548 Results.push_back(TLI.expandVectorSplice(
Node, DAG));
3552 EVT OpTy =
Node->getOperand(0).getValueType();
3553 if (
Node->getConstantOperandVal(1)) {
3557 TLI.getShiftAmountTy(
3558 Node->getOperand(0).getValueType(),
3559 DAG.getDataLayout())));
3564 Node->getOperand(0));
3572 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3573 Results.push_back(DAG.getCopyFromReg(
Node->getOperand(0), dl, SP,
3574 Node->getValueType(0)));
3577 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
3584 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3585 Results.push_back(DAG.getCopyToReg(
Node->getOperand(0), dl, SP,
3586 Node->getOperand(1)));
3592 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3607 TLI.expandIS_FPCLASS(
Node->getValueType(0),
Node->getOperand(0),
3618 switch (
Node->getOpcode()) {
3625 Tmp1 =
Node->getOperand(0);
3626 Tmp2 =
Node->getOperand(1);
3627 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3633 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG))
3639 if (
SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(
Node, DAG))
3645 EVT VT =
Node->getValueType(0);
3651 SDVTList VTs = DAG.getVTList(VT, VT);
3661 EVT VT =
Node->getValueType(0);
3665 if (TLI.getLibcallName(LC))
3671 Results.push_back(Expanded.getValue(1));
3680 if (TLI.getLibcallName(LC))
3685 Results.push_back(Expanded.getValue(1));
3693 if (
Node->getValueType(0) != MVT::f32) {
3705 if (
Node->getValueType(0) != MVT::f32) {
3710 {Node->getOperand(0), Node->getOperand(1)});
3712 {
Node->getValueType(0), MVT::Other},
3720 if (!TLI.useSoftFloat() &&
TM.Options.UnsafeFPMath) {
3722 MVT SVT =
Op.getSimpleValueType();
3723 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3729 DAG.getIntPtrConstant(0, dl,
true));
3740 DAG.shouldOptForSize()))
3741 Results.push_back(ExpandConstantFP(CFP,
true));
3746 Results.push_back(ExpandConstant(CP));
3750 EVT VT =
Node->getValueType(0);
3751 if (TLI.isOperationLegalOrCustom(
ISD::FADD, VT) &&
3752 TLI.isOperationLegalOrCustom(
ISD::FNEG, VT)) {
3761 EVT VT =
Node->getValueType(0);
3763 TLI.isOperationLegalOrCustom(
ISD::XOR, VT) &&
3764 "Don't know how to expand this subtraction!");
3765 Tmp1 = DAG.getNOT(dl,
Node->getOperand(1), VT);
3766 Tmp1 = DAG.
getNode(
ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3772 if (TLI.expandREM(
Node, Tmp1, DAG))
3779 EVT VT =
Node->getValueType(0);
3780 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3781 SDVTList VTs = DAG.getVTList(VT, VT);
3782 Tmp1 = DAG.
getNode(DivRemOpc, dl, VTs,
Node->getOperand(0),
3783 Node->getOperand(1));
3790 unsigned ExpandOpcode =
3792 EVT VT =
Node->getValueType(0);
3793 SDVTList VTs = DAG.getVTList(VT, VT);
3795 Tmp1 = DAG.
getNode(ExpandOpcode, dl, VTs,
Node->getOperand(0),
3796 Node->getOperand(1));
3804 MVT VT =
LHS.getSimpleValueType();
3805 unsigned MULHOpcode =
3808 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3810 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3816 assert(TLI.isTypeLegal(HalfType));
3817 if (TLI.expandMUL_LOHI(
Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3819 TargetLowering::MulExpansionKind::Always)) {
3820 for (
unsigned i = 0; i < 2; ++i) {
3823 SDValue Shift = DAG.getConstant(
3825 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3834 EVT VT =
Node->getValueType(0);
3835 SDVTList VTs = DAG.getVTList(VT, VT);
3841 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::SMUL_LOHI, VT);
3842 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::UMUL_LOHI, VT);
3843 bool HasMULHS = TLI.isOperationLegalOrCustom(
ISD::MULHS, VT);
3844 bool HasMULHU = TLI.isOperationLegalOrCustom(
ISD::MULHU, VT);
3845 unsigned OpToUse = 0;
3846 if (HasSMUL_LOHI && !HasMULHS) {
3848 }
else if (HasUMUL_LOHI && !HasMULHU) {
3850 }
else if (HasSMUL_LOHI) {
3852 }
else if (HasUMUL_LOHI) {
3856 Results.push_back(DAG.getNode(OpToUse, dl, VTs,
Node->getOperand(0),
3857 Node->getOperand(1)));
3865 TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
3866 TLI.isOperationLegalOrCustom(
ISD::OR, VT) &&
3867 TLI.expandMUL(
Node,
Lo,
Hi, HalfType, DAG,
3868 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3873 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3881 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG))
3886 if (
SDValue Expanded = TLI.expandROT(
Node,
true , DAG))
3893 Results.push_back(TLI.expandAddSubSat(
Node, DAG));
3907 Results.push_back(TLI.expandFixedPointMul(
Node, DAG));
3914 Node->getOperand(0),
3915 Node->getOperand(1),
3916 Node->getConstantOperandVal(2),
3939 EVT VT =
LHS.getValueType();
3943 EVT CarryType =
Node->getValueType(1);
3944 EVT SetCCType = getSetCCResultType(
Node->getValueType(0));
3946 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS,
CC);
3949 SDValue One = DAG.getConstant(1, dl, VT);
3951 DAG.
getNode(
ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3960 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero,
ISD::SETEQ)
3961 : DAG.getSetCC(dl, SetCCType, Sum, Zero,
ISD::SETEQ);
3963 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3969 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3975 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
3983 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
3991 if (TLI.expandMULO(
Node, Result, Overflow, DAG)) {
4003 DAG.getConstant(
PairTy.getSizeInBits() / 2, dl,
4004 TLI.getShiftAmountTy(
PairTy, DAG.getDataLayout())));
4009 Tmp1 =
Node->getOperand(0);
4010 Tmp2 =
Node->getOperand(1);
4011 Tmp3 =
Node->getOperand(2);
4015 cast<CondCodeSDNode>(Tmp1.
getOperand(2))->get());
4017 Tmp1 = DAG.getSelectCC(dl, Tmp1,
4028 int JTI = cast<JumpTableSDNode>(Table.
getNode())->getIndex();
4031 EVT PTy = TLI.getPointerTy(TD);
4033 unsigned EntrySize =
4034 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4041 Index = DAG.getNode(
4046 DAG.getConstant(EntrySize, dl,
Index.getValueType()));
4055 if (TLI.isJumpTableRelative()) {
4060 TLI.getPICJumpTableRelocBase(Table, DAG));
4063 Tmp1 = TLI.expandIndirectJTBranch(dl,
LD.getValue(1),
Addr, JTI, DAG);
4070 Tmp1 =
Node->getOperand(0);
4071 Tmp2 =
Node->getOperand(1);
4077 Node->getOperand(2));
4089 Node->getOperand(2));
4097 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
4102 unsigned Offset = IsStrict ? 1 : 0;
4111 bool Legalized = TLI.LegalizeSetCCCondCode(
4112 DAG,
Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4113 Chain, IsSignaling);
4121 {Chain, Tmp1, Tmp2, Tmp3},
Node->getFlags());
4125 {Tmp1, Tmp2, Tmp3, Mask, EVL},
Node->getFlags());
4127 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl,
Node->getValueType(0), Tmp1,
4128 Tmp2, Tmp3,
Node->getFlags());
4136 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->
getValueType(0));
4139 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->
getValueType(0));
4151 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
4156 EVT VT =
Node->getValueType(0);
4159 DAG.getBoolConstant(
true, dl, VT, Tmp1VT),
4160 DAG.getBoolConstant(
false, dl, VT, Tmp1VT), Tmp3);
4167 Tmp1 =
Node->getOperand(0);
4168 Tmp2 =
Node->getOperand(1);
4169 Tmp3 =
Node->getOperand(2);
4170 Tmp4 =
Node->getOperand(3);
4171 EVT VT =
Node->getValueType(0);
4181 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4183 EVT CCVT = getSetCCResultType(CmpVT);
4186 DAG.getSelect(dl, VT,
Cond, Tmp3, Tmp4,
Node->getFlags()));
4191 bool Legalized =
false;
4199 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4209 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4215 Legalized = TLI.LegalizeSetCCCondCode(
4219 assert(Legalized &&
"Can't legalize SELECT_CC with legal condition!");
4230 Tmp1, Tmp2, Tmp3, Tmp4,
CC);
4235 Tmp2, Tmp3, Tmp4,
CC);
4245 Tmp1 =
Node->getOperand(0);
4246 Tmp2 =
Node->getOperand(2);
4247 Tmp3 =
Node->getOperand(3);
4248 Tmp4 =
Node->getOperand(1);
4250 bool Legalized = TLI.LegalizeSetCCCondCode(
4251 DAG, getSetCCResultType(Tmp2.
getValueType()), Tmp2, Tmp3, Tmp4,
4254 assert(Legalized &&
"Can't legalize BR_CC with legal condition!");
4259 assert(!NeedInvert &&
"Don't know how to invert BR_CC!");
4262 Tmp4, Tmp2, Tmp3,
Node->getOperand(4));
4267 Tmp2, Tmp3,
Node->getOperand(4));
4282 EVT VT =
Node->getValueType(0);
4288 for (
unsigned Idx = 0;
Idx < NumElem;
Idx++) {
4291 Node->getOperand(0), DAG.getVectorIdxConstant(
Idx, dl));
4294 Node->getOperand(1), DAG.getVectorIdxConstant(
Idx, dl));
4318 Results.push_back(TLI.expandVecReduce(
Node, DAG));
4320 case ISD::VP_CTTZ_ELTS:
4321 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
4322 Results.push_back(TLI.expandVPCTTZElements(
Node, DAG));
4344 if (!TLI.isStrictFPEnabled() &&
Results.
empty() &&
Node->isStrictFPOpcode()) {
4350 switch (
Node->getOpcode()) {
4352 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4353 Node->getValueType(0))
4354 == TargetLowering::Legal)
4358 if (TLI.getStrictFPOperationAction(
4361 if (TLI.getStrictFPOperationAction(
4365 EVT VT =
Node->getValueType(0);
4369 {Node->getOperand(0), Node->getOperand(1), Neg},
4384 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4385 Node->getOperand(1).getValueType())
4386 == TargetLowering::Legal)
4403void SelectionDAGLegalize::ConvertNodeToLibcall(
SDNode *
Node) {
4408 unsigned Opc =
Node->getOpcode();
4417 .setChain(
Node->getOperand(0))
4420 DAG.getExternalSymbol(
"__sync_synchronize",
4421 TLI.getPointerTy(DAG.getDataLayout())),
4424 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4426 Results.push_back(CallResult.second);
4448 EVT RetVT =
Node->getValueType(0);
4451 if (TLI.getLibcallName(LC)) {
4458 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4459 "Unexpected atomic op or value type!");
4463 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4466 Node->getOperand(0));
4468 Results.push_back(Tmp.second);
4476 .setChain(
Node->getOperand(0))
4478 DAG.getExternalSymbol(
4479 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4481 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4483 Results.push_back(CallResult.second);
4491 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4492 DAG, RTLIB::CLEAR_CACHE, MVT::isVoid, {StartVal, EndVal}, CallOptions,
4494 Results.push_back(Tmp.second);
4499 ExpandFPLibCall(
Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4500 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4501 RTLIB::FMIN_PPCF128,
Results);
4508 ExpandFPLibCall(
Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4509 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4510 RTLIB::FMAX_PPCF128,
Results);
4514 ExpandFPLibCall(
Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4515 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4516 RTLIB::SQRT_PPCF128,
Results);
4519 ExpandFPLibCall(
Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4520 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4521 RTLIB::CBRT_PPCF128,
Results);
4525 ExpandFPLibCall(
Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4526 RTLIB::SIN_F80, RTLIB::SIN_F128,
4531 ExpandFPLibCall(
Node, RTLIB::COS_F32, RTLIB::COS_F64,
4532 RTLIB::COS_F80, RTLIB::COS_F128,
4537 ExpandFPLibCall(
Node, RTLIB::TAN_F32, RTLIB::TAN_F64, RTLIB::TAN_F80,
4538 RTLIB::TAN_F128, RTLIB::TAN_PPCF128,
Results);
4546 ExpandFPLibCall(
Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4547 RTLIB::LOG_F128, RTLIB::LOG_PPCF128,
Results);
4551 ExpandFPLibCall(
Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4552 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128,
Results);
4556 ExpandFPLibCall(
Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4557 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128,
Results);
4561 ExpandFPLibCall(
Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4562 RTLIB::EXP_F128, RTLIB::EXP_PPCF128,
Results);
4566 ExpandFPLibCall(
Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4567 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128,
Results);
4570 ExpandFPLibCall(
Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4571 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128,
Results);
4575 ExpandFPLibCall(
Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4576 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4577 RTLIB::TRUNC_PPCF128,
Results);
4581 ExpandFPLibCall(
Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4582 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4583 RTLIB::FLOOR_PPCF128,
Results);
4587 ExpandFPLibCall(
Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4588 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4589 RTLIB::CEIL_PPCF128,
Results);
4593 ExpandFPLibCall(
Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4594 RTLIB::RINT_F80, RTLIB::RINT_F128,
4595 RTLIB::RINT_PPCF128,
Results);
4599 ExpandFPLibCall(
Node, RTLIB::NEARBYINT_F32,
4600 RTLIB::NEARBYINT_F64,
4601 RTLIB::NEARBYINT_F80,
4602 RTLIB::NEARBYINT_F128,
4603 RTLIB::NEARBYINT_PPCF128,
Results);
4607 ExpandFPLibCall(
Node, RTLIB::ROUND_F32,
4611 RTLIB::ROUND_PPCF128,
Results);
4615 ExpandFPLibCall(
Node, RTLIB::ROUNDEVEN_F32,
4616 RTLIB::ROUNDEVEN_F64,
4617 RTLIB::ROUNDEVEN_F80,
4618 RTLIB::ROUNDEVEN_F128,
4619 RTLIB::ROUNDEVEN_PPCF128,
Results);
4623 ExpandFPLibCall(
Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4624 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128,
Results);
4633 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
4634 if (!TLI.getLibcallName(LC)) {
4636 if (
Node->isStrictFPOpcode()) {
4639 {
Node->getValueType(0),
Node->getValueType(1)},
4640 {
Node->getOperand(0),
Node->getOperand(2)});
4643 {
Node->getValueType(0),
Node->getValueType(1)},
4650 Node->getOperand(1));
4652 Node->getValueType(0),
4657 unsigned Offset =
Node->isStrictFPOpcode() ? 1 : 0;
4658 bool ExponentHasSizeOfInt =
4659 DAG.getLibInfo().getIntSize() ==
4660 Node->getOperand(1 +
Offset).getValueType().getSizeInBits();
4661 if (!ExponentHasSizeOfInt) {
4664 DAG.getContext()->emitError(
"POWI exponent does not match sizeof(int)");
4665 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
4673 ExpandFPLibCall(
Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4674 RTLIB::POW_F128, RTLIB::POW_PPCF128,
Results);
4678 ExpandArgFPLibCall(
Node, RTLIB::LROUND_F32,
4679 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4681 RTLIB::LROUND_PPCF128,
Results);
4685 ExpandArgFPLibCall(
Node, RTLIB::LLROUND_F32,
4686 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4687 RTLIB::LLROUND_F128,
4688 RTLIB::LLROUND_PPCF128,
Results);
4692 ExpandArgFPLibCall(
Node, RTLIB::LRINT_F32,
4693 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4695 RTLIB::LRINT_PPCF128,
Results);
4699 ExpandArgFPLibCall(
Node, RTLIB::LLRINT_F32,
4700 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4702 RTLIB::LLRINT_PPCF128,
Results);
4706 ExpandFPLibCall(
Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4707 RTLIB::DIV_F80, RTLIB::DIV_F128,
4712 ExpandFPLibCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
4713 RTLIB::REM_F80, RTLIB::REM_F128,
4718 ExpandFPLibCall(
Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4719 RTLIB::FMA_F80, RTLIB::FMA_F128,
4724 ExpandFPLibCall(
Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4725 RTLIB::ADD_F80, RTLIB::ADD_F128,
4730 ExpandFPLibCall(
Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4731 RTLIB::MUL_F80, RTLIB::MUL_F128,
4735 if (
Node->getValueType(0) == MVT::f32) {
4736 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32,
Node,
false).first);
4740 if (
Node->getValueType(0) == MVT::f32) {
4742 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4743 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32,
Node->getOperand(1),
4746 Results.push_back(Tmp.second);
4750 if (
Node->getValueType(0) == MVT::f32) {
4752 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4753 DAG, RTLIB::FPEXT_F16_F32, MVT::f32,
Node->getOperand(1), CallOptions,
4756 Results.push_back(Tmp.second);
4763 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16");
4764 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4770 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_bf16");
4771 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4779 bool IsStrict =
Node->isStrictFPOpcode();
4782 EVT SVT =
Node->getOperand(IsStrict ? 1 : 0).getValueType();
4783 EVT RVT =
Node->getValueType(0);
4791 for (
unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4792 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4800 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4805 NVT,
Node->getOperand(IsStrict ? 1 : 0));
4808 std::pair<SDValue, SDValue> Tmp =
4809 TLI.makeLibCall(DAG, LC, RVT,
Op, CallOptions, dl, Chain);
4812 Results.push_back(Tmp.second);
4820 bool IsStrict =
Node->isStrictFPOpcode();
4825 EVT SVT =
Op.getValueType();
4826 EVT RVT =
Node->getValueType(0);
4834 for (
unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4835 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4843 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4847 std::pair<SDValue, SDValue> Tmp =
4848 TLI.makeLibCall(DAG, LC, NVT,
Op, CallOptions, dl, Chain);
4853 Results.push_back(Tmp.second);
4864 bool IsStrict =
Node->isStrictFPOpcode();
4867 EVT VT =
Node->getValueType(0);
4868 assert(cast<ConstantSDNode>(
Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4869 "Unable to expand as libcall if it is not normal rounding");
4872 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4875 std::pair<SDValue, SDValue> Tmp =
4876 TLI.makeLibCall(DAG, LC, VT,
Op, CallOptions,
SDLoc(
Node), Chain);
4879 Results.push_back(Tmp.second);
4885 Node->getValueType(0)),
4886 Node,
false).first);
4899 Node->getValueType(0));
4901 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4904 std::pair<SDValue, SDValue> Tmp =
4905 TLI.makeLibCall(DAG, LC,
Node->getValueType(0),
Node->getOperand(1),
4908 Results.push_back(Tmp.second);
4913 ExpandFPLibCall(
Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4914 RTLIB::SUB_F80, RTLIB::SUB_F128,
4920 RTLIB::SREM_I16, RTLIB::SREM_I32,
4921 RTLIB::SREM_I64, RTLIB::SREM_I128));
4926 RTLIB::UREM_I16, RTLIB::UREM_I32,
4927 RTLIB::UREM_I64, RTLIB::UREM_I128));
4932 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4933 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4938 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4939 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4949 RTLIB::MUL_I16, RTLIB::MUL_I32,
4950 RTLIB::MUL_I64, RTLIB::MUL_I128));
4953 switch (
Node->getSimpleValueType(0).SimpleTy) {
4957 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32,
Node,
false).first);
4960 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64,
Node,
false).first);
4963 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128,
Node,
false).first);
4970 SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
4973 DAG.makeStateFunctionCall(RTLIB::FESETENV,
Ptr, Chain, dl));
4980 DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl));
4987 DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl));
4993 EVT ModeVT =
Node->getValueType(0);
4995 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4996 SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr,
4997 Node->getOperand(0), dl);
4999 ModeVT, dl, Chain, StackPtr,
5009 EVT ModeVT =
Mode.getValueType();
5011 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
5012 SDValue StInst = DAG.getStore(
5013 Node->getOperand(0), dl, Mode, StackPtr,
5016 DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl));
5024 EVT PtrTy = TLI.getPointerTy(
DL);
5026 Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
5027 Node->getOperand(0), dl));
5034 LLVM_DEBUG(
dbgs() <<
"Successfully converted node to libcall\n");
5043 MVT EltVT,
MVT NewEltVT) {
5045 MVT MidVT = OldEltsPerNewElt == 1
5052void SelectionDAGLegalize::PromoteNode(
SDNode *
Node) {
5055 MVT OVT =
Node->getSimpleValueType(0);
5061 OVT =
Node->getOperand(0).getSimpleValueType();
5068 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5069 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5070 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5071 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5072 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5073 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5074 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5075 OVT =
Node->getOperand(1).getSimpleValueType();
5078 OVT =
Node->getOperand(2).getSimpleValueType();
5079 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), OVT);
5081 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5082 switch (
Node->getOpcode()) {
5095 unsigned NewOpc =
Node->getOpcode();
5103 DAG.getConstant(TopBit, dl, NVT));
5108 Tmp1 = DAG.
getNode(NewOpc, dl, NVT, Tmp1);
5122 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5125 DAG.getConstant(DiffBits, dl,
5126 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
5139 Results.push_back(PromoteLegalFP_TO_INT_SAT(
Node, dl));
5156 &&
"VAARG promotion is supported only for vectors or integer types");
5161 Tmp1 = DAG.getVAArg(NVT, dl, Chain,
Ptr,
Node->getOperand(2),
5162 Node->getConstantOperandVal(3));
5165 Tmp2 = DAG.
getNode(TruncOp, dl, OVT, Tmp1);
5169 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), Tmp2);
5170 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
5172 UpdatedNodes->insert(Tmp2.
getNode());
5173 UpdatedNodes->insert(Chain.
getNode());
5190 unsigned ExtOp, TruncOp;
5197 switch (
Node->getOpcode()) {
5213 if (TLI.isSExtCheaperThanZExt(OVT, NVT))
5222 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5223 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5225 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5226 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
5234 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5235 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5238 auto &
DL = DAG.getDataLayout();
5242 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(
DL, NVT)));
5248 unsigned ExtOp, TruncOp;
5249 if (
Node->getValueType(0).isVector() ||
5253 }
else if (
Node->getValueType(0).isInteger()) {
5260 Tmp1 =
Node->getOperand(0);
5262 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5263 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5265 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5268 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1);
5270 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1,
5271 DAG.getIntPtrConstant(0, dl));
5283 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5292 Node->getOperand(2));
5300 MVT CVT =
Node->getSimpleValueType(0);
5301 assert(CVT == OVT &&
"not handled");
5309 if (TLI.isCondCodeLegal(CCCode, CVT)) {
5310 Tmp1 =
Node->getOperand(0);
5311 Tmp2 =
Node->getOperand(1);
5313 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5314 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5317 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5318 Tmp4 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5328 DAG.getIntPtrConstant(0, dl,
true));
5340 TLI.isSExtCheaperThanZExt(
Node->getOperand(0).getValueType(), NVT))
5345 if (
Node->isStrictFPOpcode()) {
5347 std::tie(Tmp1, std::ignore) =
5348 DAG.getStrictFPExtendOrRound(
Node->getOperand(1), InChain, dl, NVT);
5349 std::tie(Tmp2, std::ignore) =
5350 DAG.getStrictFPExtendOrRound(
Node->getOperand(2), InChain, dl, NVT);
5352 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
5353 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
5354 Results.push_back(DAG.getNode(
Node->getOpcode(), dl, VTs,
5355 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5360 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5361 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5363 Tmp2,
Node->getOperand(2),
Node->getFlags()));
5370 cast<CondCodeSDNode>(
Node->getOperand(1))->get();
5373 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5374 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5376 Node->getOperand(0),
Node->getOperand(1),
5377 Tmp1, Tmp2,
Node->getOperand(4)));
5392 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
5396 DAG.getIntPtrConstant(0, dl,
true)));
5407 {
Node->getOperand(0),
Node->getOperand(1)});
5409 {
Node->getOperand(0),
Node->getOperand(2)});
5412 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5413 {Tmp3, Tmp1, Tmp2});
5415 {Tmp1.
getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5425 DAG.getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5426 DAG.getIntPtrConstant(0, dl,
true)));
5430 {
Node->getOperand(0),
Node->getOperand(1)});
5432 {
Node->getOperand(0),
Node->getOperand(2)});
5434 {
Node->getOperand(0),
Node->getOperand(3)});
5437 Tmp4 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5438 {Tmp4, Tmp1, Tmp2, Tmp3});
5440 {Tmp4.
getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5448 Tmp2 =
Node->getOperand(1);
5449 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5459 DAG.getIntPtrConstant(isTrunc, dl,
true)));
5464 {
Node->getOperand(0),
Node->getOperand(1)});
5465 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5468 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5478 DAG.getIntPtrConstant(0, dl,
true)));
5504 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5507 DAG.getIntPtrConstant(0, dl,
true)));
5526 {
Node->getOperand(0),
Node->getOperand(1)});
5527 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5530 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5545 "Invalid promote type for build_vector");
5551 for (
unsigned I = 0, E =
Node->getNumOperands();
I != E; ++
I) {
5580 "Invalid promote type for extract_vector_elt");
5587 EVT IdxVT =
Idx.getValueType();
5589 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
5595 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5596 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5604 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
5626 "Invalid promote type for insert_vector_elt");
5634 EVT IdxVT =
Idx.getValueType();
5637 SDValue Factor = DAG.getConstant(NewEltsPerOldElt,
SDLoc(), IdxVT);
5644 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5645 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5649 CastVal, IdxOffset);
5652 NewVec, Elt, InEltIdx);
5692 "unexpected promotion type");
5694 "unexpected atomic_swap with illegal type");
5718 "unexpected promotion type");
5720 "unexpected atomic_load with illegal type");
5731 MVT ScalarType =
Scalar.getSimpleValueType();
5735 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5740 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5743 DAG.getIntPtrConstant(0, dl,
true)));
5746 case ISD::VP_REDUCE_FADD:
5747 case ISD::VP_REDUCE_FMUL:
5748 case ISD::VP_REDUCE_FMAX:
5749 case ISD::VP_REDUCE_FMIN:
5750 case ISD::VP_REDUCE_FMAXIMUM:
5751 case ISD::VP_REDUCE_FMINIMUM:
5752 case ISD::VP_REDUCE_SEQ_FADD:
5778 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes);
5785 bool AnyLegalized =
false;
5796 if (LegalizedNodes.
insert(
N).second) {
5797 AnyLegalized =
true;
5818 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes, &UpdatedNodes);
5825 return LegalizedNodes.
count(
N);
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
static MaybeAlign getAlign(Value *Ptr)
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI)
Return true if sincos libcall is available.
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
static MachineMemOperand * getStackAlignedMMO(SDValue StackPtr, MachineFunction &MF, bool isObjectScalable)
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
mir Rename Register Operands
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This is an SDNode representing atomic operations.
const SDValue & getBasePtr() const
const SDValue & getVal() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
const BasicBlock & back() const
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOStore
The memory access writes data.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetLowering & getTargetLoweringInfo() const
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
allnodes_const_iterator allnodes_end() const
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const DataLayout & getDataLayout() const
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
LLVMContext * getContext() const
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
Information about stack frame layout on the target.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
constexpr ScalarTy getFixedValue() const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ FRAME_TO_ARGS_OFFSET
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
@ RESET_FPENV
Set floating-point environment to default state.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ GLOBAL_OFFSET_TABLE
The address of the GOT.
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ SET_FPENV_MEM
Sets the current floating point environment.
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
@ Undef
Value of the register doesn't matter.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
int32_t ExponentType
A signed type to represent a floating point numbers unbiased exponent.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)