58#define DEBUG_TYPE "legalizedag"
64struct FloatSignAsInt {
87class SelectionDAGLegalize {
99 EVT getSetCCResultType(
EVT VT)
const {
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
173 void getSignAsIntValue(FloatSignAsInt &State,
const SDLoc &
DL,
175 SDValue modifySignAsInt(
const FloatSignAsInt &State,
const SDLoc &
DL,
215 dbgs() <<
" with: "; New->dump(&DAG));
218 "Replacing one node with another that produces a different number "
222 UpdatedNodes->
insert(New);
228 dbgs() <<
" with: "; New->dump(&DAG));
232 UpdatedNodes->
insert(New.getNode());
233 ReplacedNode(Old.getNode());
240 for (
unsigned i = 0, e = Old->
getNumValues(); i != e; ++i) {
244 UpdatedNodes->
insert(New[i].getNode());
251 dbgs() <<
" with: "; New->dump(&DAG));
255 UpdatedNodes->
insert(New.getNode());
256 ReplacedNode(Old.getNode());
266 bool isObjectScalable) {
268 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();
270 uint64_t ObjectSize = isObjectScalable ? ~UINT64_C(0) : MFI.getObjectSize(FI);
272 ObjectSize, MFI.getObjectAlign(FI));
279SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
284 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
286 assert(NumEltsGrowth &&
"Cannot promote to vector type with fewer elts!");
288 if (NumEltsGrowth == 1)
289 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
292 for (
unsigned i = 0; i != NumMaskElts; ++i) {
294 for (
unsigned j = 0;
j != NumEltsGrowth; ++
j) {
301 assert(NewMask.
size() == NumDestElts &&
"Non-integer NumEltsGrowth?");
302 assert(TLI.isShuffleMaskLegal(NewMask, NVT) &&
"Shuffle not legal?");
303 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
322 assert((VT == MVT::f64 || VT == MVT::f32) &&
"Invalid type expansion");
324 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
334 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
340 TLI.ShouldShrinkFPConstant(OrigVT)) {
343 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
351 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
352 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
361 OrigVT, dl, DAG.getEntryNode(), CPIdx,
369 EVT VT =
CP->getValueType(0);
370 SDValue CPIdx = DAG.getConstantPool(
CP->getConstantIntValue(),
371 TLI.getPointerTy(DAG.getDataLayout()));
372 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
374 VT, dl, DAG.getEntryNode(), CPIdx,
383SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(
SDValue Vec,
401 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
405 DAG.getEntryNode(), dl, Tmp1, StackPtr,
408 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
411 Ch = DAG.getTruncStore(
412 Ch, dl, Tmp2, StackPtr2,
416 DAG.getMachineFunction(), SPFI));
437 for (
unsigned i = 0; i != NumElts; ++i)
438 ShufOps.
push_back(i != InsertPos->getZExtValue() ? i : NumElts);
440 return DAG.getVectorShuffle(Vec.
getValueType(), dl, Vec, ScVec, ShufOps);
443 return PerformInsertVectorEltInMemory(Vec, Val,
Idx, dl);
470 TLI.isTypeLegal(MVT::i32)) {
472 bitcastToAPInt().zextOrTrunc(32),
473 SDLoc(CFP), MVT::i32);
474 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
475 ST->getOriginalAlign(), MMOFlags, AAInfo);
481 if (TLI.isTypeLegal(MVT::i64)) {
483 zextOrTrunc(64),
SDLoc(CFP), MVT::i64);
484 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
485 ST->getOriginalAlign(), MMOFlags, AAInfo);
488 if (TLI.isTypeLegal(MVT::i32) && !
ST->isVolatile()) {
495 if (DAG.getDataLayout().isBigEndian())
498 Lo = DAG.getStore(Chain, dl,
Lo,
Ptr,
ST->getPointerInfo(),
499 ST->getOriginalAlign(), MMOFlags, AAInfo);
501 Hi = DAG.getStore(Chain, dl,
Hi,
Ptr,
502 ST->getPointerInfo().getWithOffset(4),
503 ST->getOriginalAlign(), MMOFlags, AAInfo);
512void SelectionDAGLegalize::LegalizeStoreOps(
SDNode *
Node) {
521 if (!
ST->isTruncatingStore()) {
523 if (
SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
524 ReplaceNode(ST, OptStore);
529 MVT VT =
Value.getSimpleValueType();
530 switch (TLI.getOperationAction(
ISD::STORE, VT)) {
532 case TargetLowering::Legal: {
535 EVT MemVT =
ST->getMemoryVT();
537 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
538 *
ST->getMemOperand())) {
541 ReplaceNode(
SDValue(ST, 0), Result);
546 case TargetLowering::Custom: {
553 case TargetLowering::Promote: {
556 "Can only promote stores to same size type");
559 ST->getOriginalAlign(), MMOFlags, AAInfo);
569 EVT StVT =
ST->getMemoryVT();
572 auto &
DL = DAG.getDataLayout();
574 if (StWidth != StSize) {
579 Value = DAG.getZeroExtendInReg(
Value, dl, StVT);
581 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), NVT,
582 ST->getOriginalAlign(), MMOFlags, AAInfo);
588 unsigned LogStWidth =
Log2_32(StWidthBits);
590 unsigned RoundWidth = 1 << LogStWidth;
591 assert(RoundWidth < StWidthBits);
592 unsigned ExtraWidth = StWidthBits - RoundWidth;
593 assert(ExtraWidth < RoundWidth);
594 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
595 "Store size not an integral number of bytes!");
599 unsigned IncrementSize;
601 if (
DL.isLittleEndian()) {
604 Lo = DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(),
605 RoundVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
608 IncrementSize = RoundWidth / 8;
613 DAG.getConstant(RoundWidth, dl,
614 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
615 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
616 ST->getPointerInfo().getWithOffset(IncrementSize),
617 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
624 DAG.getConstant(ExtraWidth, dl,
625 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
626 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
ST->getPointerInfo(), RoundVT,
627 ST->getOriginalAlign(), MMOFlags, AAInfo);
630 IncrementSize = RoundWidth / 8;
632 DAG.getConstant(IncrementSize, dl,
633 Ptr.getValueType()));
635 ST->getPointerInfo().getWithOffset(IncrementSize),
636 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
643 switch (TLI.getTruncStoreAction(
ST->getValue().getValueType(), StVT)) {
645 case TargetLowering::Legal: {
646 EVT MemVT =
ST->getMemoryVT();
649 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
650 *
ST->getMemOperand())) {
652 ReplaceNode(
SDValue(ST, 0), Result);
656 case TargetLowering::Custom: {
662 case TargetLowering::Expand:
664 "Vector Stores are handled in LegalizeVectorOps");
669 if (TLI.isTypeLegal(StVT)) {
672 ST->getOriginalAlign(), MMOFlags, AAInfo);
677 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
680 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), StVT,
681 ST->getOriginalAlign(), MMOFlags, AAInfo);
690void SelectionDAGLegalize::LegalizeLoadOps(
SDNode *
Node) {
699 LLVM_DEBUG(
dbgs() <<
"Legalizing non-extending load operation\n");
700 MVT VT =
Node->getSimpleValueType(0);
704 switch (TLI.getOperationAction(
Node->getOpcode(), VT)) {
706 case TargetLowering::Legal: {
707 EVT MemVT =
LD->getMemoryVT();
711 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
712 *
LD->getMemOperand())) {
713 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
717 case TargetLowering::Custom:
718 if (
SDValue Res = TLI.LowerOperation(RVal, DAG)) {
724 case TargetLowering::Promote: {
725 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
727 "Can only promote loads to same size type");
729 SDValue Res = DAG.getLoad(NVT, dl, Chain,
Ptr,
LD->getMemOperand());
737 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), RVal);
738 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), RChain);
740 UpdatedNodes->insert(RVal.
getNode());
741 UpdatedNodes->insert(RChain.
getNode());
749 EVT SrcVT =
LD->getMemoryVT();
763 TLI.getLoadExtAction(ExtType,
Node->getValueType(0), MVT::i1) ==
764 TargetLowering::Promote)) {
778 Chain,
Ptr,
LD->getPointerInfo(), NVT,
779 LD->getOriginalAlign(), MMOFlags, AAInfo);
787 Result, DAG.getValueType(SrcVT));
791 Result.getValueType(), Result,
792 DAG.getValueType(SrcVT));
800 unsigned LogSrcWidth =
Log2_32(SrcWidthBits);
802 unsigned RoundWidth = 1 << LogSrcWidth;
803 assert(RoundWidth < SrcWidthBits);
804 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
805 assert(ExtraWidth < RoundWidth);
806 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
807 "Load size not an integral number of bytes!");
811 unsigned IncrementSize;
812 auto &
DL = DAG.getDataLayout();
814 if (
DL.isLittleEndian()) {
818 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
822 IncrementSize = RoundWidth / 8;
825 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
826 LD->getPointerInfo().getWithOffset(IncrementSize),
827 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
837 DAG.getConstant(RoundWidth, dl,
838 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
846 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
847 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
851 IncrementSize = RoundWidth / 8;
855 LD->getPointerInfo().getWithOffset(IncrementSize),
856 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
866 DAG.getConstant(ExtraWidth, dl,
867 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
875 bool isCustom =
false;
876 switch (TLI.getLoadExtAction(ExtType,
Node->getValueType(0),
879 case TargetLowering::Custom:
882 case TargetLowering::Legal:
894 EVT MemVT =
LD->getMemoryVT();
896 if (!TLI.allowsMemoryAccess(*DAG.getContext(),
DL, MemVT,
897 *
LD->getMemOperand())) {
898 std::tie(
Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
903 case TargetLowering::Expand: {
904 EVT DestVT =
Node->getValueType(0);
910 (TLI.isTypeLegal(SrcVT) ||
911 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
918 SrcVT,
LD->getMemOperand());
921 Value = DAG.getNode(ExtendOp, dl,
Node->getValueType(0), Load);
922 Chain =
Load.getValue(1);
931 if (SVT == MVT::f16 || SVT == MVT::bf16) {
937 Ptr, ISrcVT,
LD->getMemOperand());
941 Chain =
Result.getValue(1);
947 "Vector Loads are handled in LegalizeVectorOps");
954 "EXTLOAD should always be supported!");
958 Node->getValueType(0),
960 LD->getMemOperand());
965 Result, DAG.getValueType(SrcVT));
967 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
969 Chain =
Result.getValue(1);
980 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
982 UpdatedNodes->insert(
Value.getNode());
983 UpdatedNodes->insert(Chain.
getNode());
990void SelectionDAGLegalize::LegalizeOp(
SDNode *
Node) {
999 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
1000 assert(TLI.getTypeAction(*DAG.getContext(),
Node->getValueType(i)) ==
1001 TargetLowering::TypeLegal &&
1002 "Unexpected illegal type!");
1005 assert((TLI.getTypeAction(*DAG.getContext(),
Op.getValueType()) ==
1006 TargetLowering::TypeLegal ||
1009 "Unexpected illegal type!");
1014 bool SimpleFinishLegalizing =
true;
1015 switch (
Node->getOpcode()) {
1020 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
1023 Action = TLI.getOperationAction(
Node->getOpcode(),
1024 Node->getValueType(0));
1027 Action = TLI.getOperationAction(
Node->getOpcode(),
1028 Node->getValueType(0));
1029 if (Action != TargetLowering::Promote)
1030 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
1034 Action = TLI.getOperationAction(
Node->getOpcode(),
1035 Node->getOperand(1).getValueType());
1046 Action = TLI.getOperationAction(
Node->getOpcode(),
1047 Node->getOperand(0).getValueType());
1060 Action = TLI.getOperationAction(
Node->getOpcode(),
1061 Node->getOperand(1).getValueType());
1064 EVT InnerType = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1065 Action = TLI.getOperationAction(
Node->getOpcode(), InnerType);
1069 Action = TLI.getOperationAction(
Node->getOpcode(),
1070 Node->getOperand(1).getValueType());
1079 unsigned Opc =
Node->getOpcode();
1084 : (Opc ==
ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1086 unsigned CompareOperand = Opc ==
ISD::BR_CC ? 2
1090 MVT OpVT =
Node->getOperand(CompareOperand).getSimpleValueType();
1092 cast<CondCodeSDNode>(
Node->getOperand(CCOperand))->get();
1093 Action = TLI.getCondCodeAction(CCCode, OpVT);
1094 if (Action == TargetLowering::Legal) {
1096 Action = TLI.getOperationAction(
Node->getOpcode(),
1097 Node->getValueType(0));
1099 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
1107 SimpleFinishLegalizing =
false;
1114 SimpleFinishLegalizing =
false;
1127 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1128 if (Action == TargetLowering::Legal)
1129 Action = TargetLowering::Expand;
1139 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1140 if (Action == TargetLowering::Legal)
1141 Action = TargetLowering::Custom;
1147 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::i64);
1154 Action = TargetLowering::Legal;
1157 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1158 if (Action == TargetLowering::Expand) {
1162 Node->getOperand(0));
1169 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1170 if (Action == TargetLowering::Expand) {
1174 Node->getOperand(0));
1188 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1198 unsigned Scale =
Node->getConstantOperandVal(2);
1199 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
1200 Node->getValueType(0), Scale);
1204 Action = TLI.getOperationAction(
Node->getOpcode(),
1205 cast<MaskedScatterSDNode>(
Node)->getValue().getValueType());
1208 Action = TLI.getOperationAction(
Node->getOpcode(),
1209 cast<MaskedStoreSDNode>(
Node)->getValue().getValueType());
1211 case ISD::VP_SCATTER:
1212 Action = TLI.getOperationAction(
1214 cast<VPScatterSDNode>(
Node)->getValue().getValueType());
1217 Action = TLI.getOperationAction(
1219 cast<VPStoreSDNode>(
Node)->getValue().getValueType());
1221 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1222 Action = TLI.getOperationAction(
1224 cast<VPStridedStoreSDNode>(
Node)->getValue().getValueType());
1242 Action = TLI.getOperationAction(
1243 Node->getOpcode(),
Node->getOperand(0).getValueType());
1247 case ISD::VP_REDUCE_FADD:
1248 case ISD::VP_REDUCE_FMUL:
1249 case ISD::VP_REDUCE_ADD:
1250 case ISD::VP_REDUCE_MUL:
1251 case ISD::VP_REDUCE_AND:
1252 case ISD::VP_REDUCE_OR:
1253 case ISD::VP_REDUCE_XOR:
1254 case ISD::VP_REDUCE_SMAX:
1255 case ISD::VP_REDUCE_SMIN:
1256 case ISD::VP_REDUCE_UMAX:
1257 case ISD::VP_REDUCE_UMIN:
1258 case ISD::VP_REDUCE_FMAX:
1259 case ISD::VP_REDUCE_FMIN:
1260 case ISD::VP_REDUCE_SEQ_FADD:
1261 case ISD::VP_REDUCE_SEQ_FMUL:
1262 Action = TLI.getOperationAction(
1263 Node->getOpcode(),
Node->getOperand(1).getValueType());
1267 Action = TLI.getCustomOperationAction(*
Node);
1269 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1274 if (SimpleFinishLegalizing) {
1276 switch (
Node->getOpcode()) {
1295 NewNode = DAG.UpdateNodeOperands(
Node, Op0, SAO);
1315 NewNode = DAG.UpdateNodeOperands(
Node, Op0, Op1, SAO);
1321 if (NewNode !=
Node) {
1322 ReplaceNode(
Node, NewNode);
1326 case TargetLowering::Legal:
1329 case TargetLowering::Custom:
1337 if (
Node->getNumValues() == 1) {
1341 Node->getValueType(0) == MVT::Glue) &&
1342 "Type mismatch for custom legalized operation");
1350 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i) {
1354 Node->getValueType(i) == MVT::Glue) &&
1355 "Type mismatch for custom legalized operation");
1359 ReplaceNode(
Node, ResultVals.
data());
1364 case TargetLowering::Expand:
1365 if (ExpandNode(
Node))
1368 case TargetLowering::LibCall:
1369 ConvertNodeToLibcall(
Node);
1371 case TargetLowering::Promote:
1377 switch (
Node->getOpcode()) {
1390 return LegalizeLoadOps(
Node);
1392 return LegalizeStoreOps(
Node);
1396SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(
SDValue Op) {
1416 if (
ST->isIndexed() ||
ST->isTruncatingStore() ||
1417 ST->getValue() != Vec)
1422 if (!
ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1431 ST->hasPredecessor(
Op.getNode()))
1444 StackPtr = DAG.CreateStackTemporary(VecVT);
1447 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO);
1451 Align ElementAlignment =
1452 std::min(cast<StoreSDNode>(Ch)->
getAlign(),
1453 DAG.getDataLayout().getPrefTypeAlign(
1454 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1456 if (
Op.getValueType().isVector()) {
1457 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1458 Op.getValueType(),
Idx);
1459 NewLoad = DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr,
1462 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1463 NewLoad = DAG.getExtLoad(
ISD::EXTLOAD, dl,
Op.getValueType(), Ch, StackPtr,
1469 DAG.ReplaceAllUsesOfValueWith(Ch,
SDValue(NewLoad.
getNode(), 1));
1475 NewLoadOperands[0] = Ch;
1477 SDValue(DAG.UpdateNodeOperands(NewLoad.
getNode(), NewLoadOperands), 0);
1481SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(
SDValue Op) {
1482 assert(
Op.getValueType().isVector() &&
"Non-vector insert subvector!");
1493 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1498 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1502 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT,
Idx);
1506 Ch, dl, Part, SubStackPtr,
1510 return DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1516 "Unexpected opcode!");
1522 EVT VT =
Node->getValueType(0);
1524 :
Node->getOperand(0).getValueType();
1526 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1527 int FI = cast<FrameIndexSDNode>(FIPtr.
getNode())->getIndex();
1534 assert(TypeByteSize > 0 &&
"Vector element type too small for stack store!");
1538 bool Truncate = isa<BuildVectorSDNode>(
Node) &&
1539 MemVT.
bitsLT(
Node->getOperand(0).getValueType());
1542 for (
unsigned i = 0, e =
Node->getNumOperands(); i != e; ++i) {
1544 if (
Node->getOperand(i).isUndef())
continue;
1546 unsigned Offset = TypeByteSize*i;
1552 Stores.
push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1556 Stores.
push_back(DAG.getStore(DAG.getEntryNode(), dl,
Node->getOperand(i),
1561 if (!Stores.
empty())
1564 StoreChain = DAG.getEntryNode();
1567 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1573void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1576 EVT FloatVT =
Value.getValueType();
1578 State.FloatVT = FloatVT;
1581 if (TLI.isTypeLegal(IVT)) {
1584 State.SignBit = NumBits - 1;
1590 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1593 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1598 State.Chain = DAG.getStore(DAG.getEntryNode(),
DL,
Value, State.FloatPtr,
1599 State.FloatPointerInfo);
1606 State.IntPointerInfo = State.FloatPointerInfo;
1609 unsigned ByteOffset = (NumBits / 8) - 1;
1616 State.IntPtr = IntPtr;
1617 State.IntValue = DAG.getExtLoad(
ISD::EXTLOAD,
DL, LoadTy, State.Chain, IntPtr,
1618 State.IntPointerInfo, MVT::i8);
1625SDValue SelectionDAGLegalize::modifySignAsInt(
const FloatSignAsInt &State,
1632 SDValue Chain = DAG.getTruncStore(State.Chain,
DL, NewIntValue, State.IntPtr,
1633 State.IntPointerInfo, MVT::i8);
1634 return DAG.getLoad(State.FloatVT,
DL, Chain, State.FloatPtr,
1635 State.FloatPointerInfo);
1644 FloatSignAsInt SignAsInt;
1645 getSignAsIntValue(SignAsInt,
DL, Sign);
1647 EVT IntVT = SignAsInt.IntValue.getValueType();
1648 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1654 if (TLI.isOperationLegalOrCustom(
ISD::FABS, FloatVT) &&
1655 TLI.isOperationLegalOrCustom(
ISD::FNEG, FloatVT)) {
1658 SDValue Cond = DAG.getSetCC(
DL, getSetCCResultType(IntVT), SignBit,
1660 return DAG.getSelect(
DL, FloatVT,
Cond, NegValue, AbsValue);
1664 FloatSignAsInt MagAsInt;
1665 getSignAsIntValue(MagAsInt,
DL, Mag);
1666 EVT MagVT = MagAsInt.IntValue.getValueType();
1667 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask,
DL, MagVT);
1672 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1673 EVT ShiftVT = IntVT;
1679 if (ShiftAmount > 0) {
1680 SDValue ShiftCnst = DAG.getConstant(ShiftAmount,
DL, ShiftVT);
1682 }
else if (ShiftAmount < 0) {
1683 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount,
DL, ShiftVT);
1693 return modifySignAsInt(MagAsInt,
DL, CopiedSign);
1699 FloatSignAsInt SignAsInt;
1700 getSignAsIntValue(SignAsInt,
DL,
Node->getOperand(0));
1701 EVT IntVT = SignAsInt.IntValue.getValueType();
1704 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1709 return modifySignAsInt(SignAsInt,
DL, SignFlip);
1717 EVT FloatVT =
Value.getValueType();
1724 FloatSignAsInt ValueAsInt;
1725 getSignAsIntValue(ValueAsInt,
DL,
Value);
1726 EVT IntVT = ValueAsInt.IntValue.getValueType();
1727 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask,
DL, IntVT);
1730 return modifySignAsInt(ValueAsInt,
DL, ClearedSign);
1733void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(
SDNode*
Node,
1735 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1736 assert(SPReg &&
"Target cannot require DYNAMIC_STACKALLOC expansion and"
1737 " not tell us which reg is the stack pointer!");
1739 EVT VT =
Node->getValueType(0);
1747 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1750 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1752 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1760 if (Alignment > StackAlign)
1762 DAG.getConstant(-Alignment.
value(), dl, VT));
1763 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);
1765 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0,
SDValue(), dl);
1777 return EmitStackConvert(
SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1785 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1788 if ((SrcVT.
bitsGT(SlotVT) &&
1789 !TLI.isTruncStoreLegalOrCustom(
SrcOp.getValueType(), SlotVT)) ||
1790 (SlotVT.
bitsLT(DestVT) &&
1791 !TLI.isLoadExtLegalOrCustom(
ISD::EXTLOAD, DestVT, SlotVT)))
1795 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1796 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1808 if (SrcVT.
bitsGT(SlotVT))
1809 Store = DAG.getTruncStore(Chain, dl,
SrcOp, FIPtr, PtrInfo,
1813 Store = DAG.getStore(Chain, dl,
SrcOp, FIPtr, PtrInfo, SrcAlign);
1817 if (SlotVT.
bitsEq(DestVT))
1818 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1821 return DAG.getExtLoad(
ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1834 SDValue Ch = DAG.getTruncStore(
1835 DAG.getEntryNode(), dl,
Node->getOperand(0), StackPtr,
1837 Node->getValueType(0).getVectorElementType());
1839 Node->getValueType(0), dl, Ch, StackPtr,
1846 unsigned NumElems =
Node->getNumOperands();
1848 EVT VT =
Node->getValueType(0);
1860 for (
unsigned i = 0; i < NumElems; ++i) {
1871 while (IntermedVals.
size() > 2) {
1872 NewIntermedVals.
clear();
1873 for (
unsigned i = 0, e = (IntermedVals.
size() & ~1u); i < e; i += 2) {
1879 FinalIndices.
reserve(IntermedVals[i].second.
size() +
1880 IntermedVals[i+1].second.
size());
1883 for (
unsigned j = 0, f = IntermedVals[i].second.
size(); j != f;
1886 FinalIndices.
push_back(IntermedVals[i].second[j]);
1888 for (
unsigned j = 0, f = IntermedVals[i+1].second.
size(); j != f;
1890 ShuffleVec[k] = NumElems + j;
1891 FinalIndices.
push_back(IntermedVals[i+1].second[j]);
1897 IntermedVals[i+1].first,
1902 std::make_pair(Shuffle, std::move(FinalIndices)));
1907 if ((IntermedVals.
size() & 1) != 0)
1910 IntermedVals.
swap(NewIntermedVals);
1914 "Invalid number of intermediate vectors");
1915 SDValue Vec1 = IntermedVals[0].first;
1917 if (IntermedVals.
size() > 1)
1918 Vec2 = IntermedVals[1].first;
1923 for (
unsigned i = 0, e = IntermedVals[0].second.
size(); i != e; ++i)
1924 ShuffleVec[IntermedVals[0].second[i]] = i;
1925 for (
unsigned i = 0, e = IntermedVals[1].second.
size(); i != e; ++i)
1926 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1940 unsigned NumElems =
Node->getNumOperands();
1943 EVT VT =
Node->getValueType(0);
1944 EVT OpVT =
Node->getOperand(0).getValueType();
1949 bool isOnlyLowElement =
true;
1950 bool MoreThanTwoValues =
false;
1952 for (
unsigned i = 0; i < NumElems; ++i) {
1957 isOnlyLowElement =
false;
1958 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1963 }
else if (!Value2.
getNode()) {
1966 }
else if (V != Value1 && V != Value2) {
1967 MoreThanTwoValues =
true;
1972 return DAG.getUNDEF(VT);
1974 if (isOnlyLowElement)
1980 for (
unsigned i = 0, e = NumElems; i !=
e; ++i) {
1982 dyn_cast<ConstantFPSDNode>(
Node->getOperand(i))) {
1985 dyn_cast<ConstantSDNode>(
Node->getOperand(i))) {
2004 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
2005 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2007 VT, dl, DAG.getEntryNode(), CPIdx,
2013 for (
unsigned i = 0; i < NumElems; ++i) {
2014 if (
Node->getOperand(i).isUndef())
2019 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.
size())) {
2020 if (!MoreThanTwoValues) {
2022 for (
unsigned i = 0; i < NumElems; ++i) {
2026 ShuffleVec[i] =
V == Value1 ? 0 : NumElems;
2028 if (TLI.isShuffleMaskLegal(ShuffleVec,
Node->getValueType(0))) {
2035 Vec2 = DAG.getUNDEF(VT);
2038 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2048 return ExpandVectorBuildThroughStack(
Node);
2053 EVT VT =
Node->getValueType(0);
2056 return DAG.getSplatBuildVector(VT,
DL, SplatVal);
2067 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2068 TLI.getPointerTy(DAG.getDataLayout()));
2070 EVT RetVT =
Node->getValueType(0);
2077 SDValue InChain = DAG.getEntryNode();
2082 const Function &
F = DAG.getMachineFunction().getFunction();
2084 TLI.isInTailCallPosition(DAG,
Node, TCChain) &&
2085 (
RetTy ==
F.getReturnType() ||
F.getReturnType()->isVoidTy());
2090 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT,
isSigned);
2093 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2095 .setTailCall(isTailCall)
2096 .setSExtResult(signExtend)
2097 .setZExtResult(!signExtend)
2098 .setIsPostTypeLegalization(
true);
2100 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2105 return {DAG.getRoot(), DAG.getRoot()};
2117 EVT ArgVT =
Op.getValueType();
2121 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT,
isSigned);
2122 Entry.IsZExt = !Entry.IsSExt;
2123 Args.push_back(Entry);
2126 return ExpandLibCall(LC,
Node, std::move(Args),
isSigned);
2129void SelectionDAGLegalize::ExpandFrexpLibCall(
2132 EVT VT =
Node->getValueType(0);
2133 EVT ExpVT =
Node->getValueType(1);
2141 FPArgEntry.
Node = FPOp;
2142 FPArgEntry.
Ty = ArgTy;
2144 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2146 PtrArgEntry.
Node = StackSlot;
2147 PtrArgEntry.
Ty = PointerType::get(*DAG.getContext(),
2148 DAG.getDataLayout().getAllocaAddrSpace());
2153 auto [
Call, Chain] = ExpandLibCall(LC,
Node, std::move(Args),
false);
2157 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2161 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2163 LoadExp.
getValue(1), DAG.getRoot());
2164 DAG.setRoot(OutputChain);
2170void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2173 if (LC == RTLIB::UNKNOWN_LIBCALL)
2176 if (
Node->isStrictFPOpcode()) {
2177 EVT RetVT =
Node->getValueType(0);
2181 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2184 Node->getOperand(0));
2186 Results.push_back(Tmp.second);
2188 SDValue Tmp = ExpandLibCall(LC,
Node,
false).first;
2194void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2202 Call_F32, Call_F64, Call_F80,
2203 Call_F128, Call_PPCF128);
2214 switch (
Node->getSimpleValueType(0).SimpleTy) {
2216 case MVT::i8: LC = Call_I8;
break;
2217 case MVT::i16: LC = Call_I16;
break;
2218 case MVT::i32: LC = Call_I32;
break;
2219 case MVT::i64: LC = Call_I64;
break;
2220 case MVT::i128: LC = Call_I128;
break;
2227void SelectionDAGLegalize::ExpandArgFPLibCall(
SDNode*
Node,
2234 EVT InVT =
Node->getOperand(
Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2236 Call_F32, Call_F64, Call_F80,
2237 Call_F128, Call_PPCF128);
2243SelectionDAGLegalize::ExpandDivRemLibCall(
SDNode *
Node,
2245 unsigned Opcode =
Node->getOpcode();
2249 switch (
Node->getSimpleValueType(0).SimpleTy) {
2251 case MVT::i8: LC=
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
2252 case MVT::i16: LC=
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
2253 case MVT::i32: LC=
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
2254 case MVT::i64: LC=
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
2255 case MVT::i128: LC=
isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
2261 SDValue InChain = DAG.getEntryNode();
2263 EVT RetVT =
Node->getValueType(0);
2269 EVT ArgVT =
Op.getValueType();
2275 Args.push_back(Entry);
2279 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2281 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2284 Args.push_back(Entry);
2286 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2287 TLI.getPointerTy(DAG.getDataLayout()));
2293 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2298 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2310 switch (
Node->getSimpleValueType(0).SimpleTy) {
2312 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2313 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2314 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2315 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2316 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2339SelectionDAGLegalize::ExpandSinCosLibCall(
SDNode *
Node,
2342 switch (
Node->getSimpleValueType(0).SimpleTy) {
2344 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2345 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2346 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2347 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2348 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2354 SDValue InChain = DAG.getEntryNode();
2356 EVT RetVT =
Node->getValueType(0);
2363 Entry.Node =
Node->getOperand(0);
2365 Entry.IsSExt =
false;
2366 Entry.IsZExt =
false;
2367 Args.push_back(Entry);
2370 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2371 Entry.Node = SinPtr;
2372 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2373 Entry.IsSExt =
false;
2374 Entry.IsZExt =
false;
2375 Args.push_back(Entry);
2378 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2379 Entry.Node = CosPtr;
2380 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2381 Entry.IsSExt =
false;
2382 Entry.IsZExt =
false;
2383 Args.push_back(Entry);
2385 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2386 TLI.getPointerTy(DAG.getDataLayout()));
2390 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2391 TLI.getLibcallCallingConv(LC),
Type::getVoidTy(*DAG.getContext()), Callee,
2394 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2404 EVT VT =
Node->getValueType(0);
2407 EVT ExpVT =
N.getValueType();
2409 if (AsIntVT ==
EVT())
2422 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2427 const int Precision = APFloat::semanticsPrecision(FltSem);
2429 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2430 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2432 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2434 const APFloat One(FltSem,
"1.0");
2435 APFloat ScaleUpK =
scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2439 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2448 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2454 DAG.getSetCC(dl, SetCCVT,
N, DoubleMaxExp,
ISD::SETUGT);
2456 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2468 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2469 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2474 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2479 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2483 SDValue ScaleDownTwice = DAG.getSetCC(
2484 dl, SetCCVT,
N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2496 DAG.getNode(
ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small,
X));
2500 DAG.getNode(
ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small,
N));
2505 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2506 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2509 ExponentShiftAmt, NUW_NSW);
2511 return DAG.getNode(
ISD::FMUL, dl, VT, NewX, AsFP);
2518 EVT ExpVT =
Node->getValueType(1);
2520 if (AsIntVT ==
EVT())
2525 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2542 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2546 SDValue SmallestNormalizedInt = DAG.getConstant(
2552 DAG.getConstant(
APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2558 FractSignMaskVal.
setBit(BitSize - 1);
2561 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2563 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2565 const APFloat One(FltSem,
"1.0");
2569 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2571 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2575 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2581 SDValue AddNegSmallestNormal =
2583 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2587 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt,
ISD::SETULT);
2589 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2604 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2607 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2610 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2616 const APFloat Half(FltSem,
"0.5");
2617 SDValue FPHalf = DAG.getConstant(Half.bitcastToAPInt(), dl, AsIntVT);
2630 return DAG.getMergeValues({Result0, Result1}, dl);
2641 EVT DestVT =
Node->getValueType(0);
2643 unsigned OpNo =
Node->isStrictFPOpcode() ? 1 : 0;
2649 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2650 (DestVT.
bitsLE(MVT::f64) ||
2654 LLVM_DEBUG(
dbgs() <<
"32-bit [signed|unsigned] integer to float/double "
2658 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2665 DAG.getConstant(0x80000000u, dl, MVT::i32));
2668 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2671 if (DAG.getDataLayout().isBigEndian())
2674 SDValue MemChain = DAG.getEntryNode();
2677 SDValue Store1 = DAG.getStore(MemChain, dl,
Lo, StackSlot,
2690 SDValue Bias = DAG.getConstantFP(
2691 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2692 : llvm::bit_cast<double>(0x4330000000000000ULL),
2697 if (
Node->isStrictFPOpcode()) {
2699 {
Node->getOperand(0), Load, Bias});
2702 std::pair<SDValue, SDValue> ResultPair;
2704 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2705 Result = ResultPair.first;
2706 Chain = ResultPair.second;
2712 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2721 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2722 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2723 LLVM_DEBUG(
dbgs() <<
"Converting unsigned i32/i64 to f32/f64\n");
2738 EVT SetCCVT = getSetCCResultType(SrcVT);
2740 SDValue SignBitTest = DAG.getSetCC(
2741 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT),
ISD::SETLT);
2743 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2744 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2746 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2751 if (
Node->isStrictFPOpcode()) {
2754 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest,
Or, Op0);
2756 {
Node->getOperand(0), InCvt });
2764 Flags.setNoFPExcept(
Node->getFlags().hasNoFPExcept());
2765 Fast->setFlags(Flags);
2766 Flags.setNoFPExcept(
true);
2774 return DAG.getSelect(dl, DestVT, SignBitTest, Slow,
Fast);
2778 if (!TLI.isOperationLegalOrCustom(
2785 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2787 "Cannot perform lossless SINT_TO_FP!");
2790 if (
Node->isStrictFPOpcode()) {
2792 {
Node->getOperand(0), Op0 });
2796 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2799 Four = DAG.getIntPtrConstant(4, dl);
2800 SDValue CstOffset = DAG.getSelect(dl,
Zero.getValueType(),
2801 SignSet, Four, Zero);
2810 case MVT::i8 : FF = 0x43800000ULL;
break;
2811 case MVT::i16: FF = 0x47800000ULL;
break;
2812 case MVT::i32: FF = 0x4F800000ULL;
break;
2813 case MVT::i64: FF = 0x5F800000ULL;
break;
2815 if (DAG.getDataLayout().isLittleEndian())
2817 Constant *FudgeFactor = ConstantInt::get(
2821 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2822 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2826 if (DestVT == MVT::f32)
2827 FudgeInReg = DAG.getLoad(
2828 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2837 LegalizeOp(
Load.getNode());
2841 if (
Node->isStrictFPOpcode()) {
2843 { Tmp1.
getValue(1), Tmp1, FudgeInReg });
2844 Chain =
Result.getValue(1);
2848 return DAG.getNode(
ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2856void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2858 bool IsStrict =
N->isStrictFPOpcode();
2861 EVT DestVT =
N->getValueType(0);
2862 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2869 unsigned OpToUse = 0;
2877 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2885 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2897 DAG.
getNode(OpToUse, dl, {DestVT, MVT::Other},
2900 dl, NewInTy, LegalOp)});
2907 DAG.getNode(OpToUse, dl, DestVT,
2909 dl, NewInTy, LegalOp)));
2917void SelectionDAGLegalize::PromoteLegalFP_TO_INT(
SDNode *
N,
const SDLoc &dl,
2919 bool IsStrict =
N->isStrictFPOpcode();
2922 EVT DestVT =
N->getValueType(0);
2923 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2925 EVT NewOutTy = DestVT;
2927 unsigned OpToUse = 0;
2937 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2942 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2951 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2952 Operation = DAG.getNode(OpToUse, dl, VTs,
N->getOperand(0), LegalOp);
2954 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2969 unsigned Opcode =
Node->getOpcode();
2972 EVT NewOutTy =
Node->getValueType(0);
2977 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2984 Node->getOperand(1));
2990 EVT VT =
Op.getValueType();
2991 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2996 if (TLI.isOperationLegalOrPromote(
ISD::CTPOP, VT)) {
3002 DAG.getConstant(1ULL << (--i), dl, ShVT));
3007 return DAG.getNode(
ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
3010bool SelectionDAGLegalize::ExpandNode(
SDNode *
Node) {
3014 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3016 switch (
Node->getOpcode()) {
3018 if ((Tmp1 = TLI.expandABS(
Node, DAG)))
3023 if ((Tmp1 = TLI.expandABD(
Node, DAG)))
3027 if ((Tmp1 = TLI.expandCTPOP(
Node, DAG)))
3032 if ((Tmp1 = TLI.expandCTLZ(
Node, DAG)))
3037 if ((Tmp1 = TLI.expandCTTZ(
Node, DAG)))
3041 if ((Tmp1 = TLI.expandBITREVERSE(
Node, DAG)))
3045 if ((Tmp1 = TLI.expandBSWAP(
Node, DAG)))
3049 Results.push_back(ExpandPARITY(
Node->getOperand(0), dl));
3054 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3057 SDValue CfaArg = DAG.getSExtOrTrunc(
Node->getOperand(0), dl,
3058 TLI.getPointerTy(DAG.getDataLayout()));
3066 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3072 Results.push_back(DAG.getConstant(1, dl,
Node->getValueType(0)));
3089 DAG.getConstant(0, dl,
Node->getValueType(0)));
3095 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3101 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3102 SDValue Swap = DAG.getAtomicCmpSwap(
3104 Node->getOperand(0),
Node->getOperand(1), Zero, Zero,
3105 cast<AtomicSDNode>(
Node)->getMemOperand());
3114 Node->getOperand(0),
Node->getOperand(2),
Node->getOperand(1),
3115 cast<AtomicSDNode>(
Node)->getMemOperand());
3123 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3124 SDValue Res = DAG.getAtomicCmpSwap(
3126 Node->getOperand(0),
Node->getOperand(1),
Node->getOperand(2),
3127 Node->getOperand(3), cast<MemSDNode>(
Node)->getMemOperand());
3133 EVT AtomicType = cast<AtomicSDNode>(
Node)->getMemoryVT();
3134 EVT OuterType =
Node->getValueType(0);
3135 switch (TLI.getExtendForAtomicOps()) {
3138 DAG.getValueType(AtomicType));
3140 Node->getOperand(2), DAG.getValueType(AtomicType));
3145 DAG.getValueType(AtomicType));
3146 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3150 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3151 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3167 EVT VT =
Node->getValueType(0);
3171 cast<VTSDNode>(
RHS->getOperand(1))->getVT() == AN->
getMemoryVT())
3172 RHS =
RHS->getOperand(0);
3176 Node->getOperand(0),
Node->getOperand(1),
3186 for (
unsigned i = 0; i <
Node->getNumValues(); i++)
3190 EVT VT =
Node->getValueType(0);
3192 Results.push_back(DAG.getConstant(0, dl, VT));
3195 Results.push_back(DAG.getConstantFP(0, dl, VT));
3202 if (TLI.isStrictFPEnabled())
3206 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3207 Node->getValueType(0))
3208 == TargetLowering::Legal)
3212 if ((Tmp1 = EmitStackConvert(
Node->getOperand(1),
Node->getValueType(0),
3213 Node->getValueType(0), dl,
3214 Node->getOperand(0)))) {
3216 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_ROUND node\n");
3221 if ((Tmp1 = TLI.expandFP_ROUND(
Node, DAG))) {
3229 if ((Tmp1 = EmitStackConvert(
Node->getOperand(0),
Node->getValueType(0),
3230 Node->getValueType(0), dl)))
3236 if (TLI.isStrictFPEnabled())
3240 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3241 Node->getValueType(0))
3242 == TargetLowering::Legal)
3246 if ((Tmp1 = EmitStackConvert(
3247 Node->getOperand(1),
Node->getOperand(1).getValueType(),
3248 Node->getValueType(0), dl,
Node->getOperand(0)))) {
3250 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_EXTEND node\n");
3256 EVT SrcVT =
Op.getValueType();
3257 EVT DstVT =
Node->getValueType(0);
3263 if ((Tmp1 = EmitStackConvert(
Op, SrcVT, DstVT, dl)))
3273 if (
Op.getValueType() == MVT::bf16) {
3277 Op = DAG.getAnyExtOrTrunc(
Op, dl, MVT::i32);
3281 DAG.getConstant(16, dl,
3282 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3285 if (
Node->getValueType(0) != MVT::f32)
3292 if (
Op.getValueType() != MVT::f32)
3294 DAG.getIntPtrConstant(0, dl,
true));
3296 if (!DAG.isKnownNeverSNaN(
Op)) {
3301 DAG.getConstant(16, dl,
3302 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3305 if (
Node->getValueType(0) == MVT::bf16) {
3309 Op = DAG.getAnyExtOrTrunc(
Op, dl,
Node->getValueType(0));
3315 EVT ExtraVT = cast<VTSDNode>(
Node->getOperand(1))->getVT();
3316 EVT VT =
Node->getValueType(0);
3326 SDValue One = DAG.getConstant(1, dl, VT);
3336 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3339 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3341 Node->getOperand(0), ShiftCst);
3348 if (TLI.expandUINT_TO_FP(
Node, Tmp1, Tmp2, DAG)) {
3350 if (
Node->isStrictFPOpcode())
3357 if ((Tmp1 = ExpandLegalINT_TO_FP(
Node, Tmp2))) {
3359 if (
Node->isStrictFPOpcode())
3364 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG))
3368 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG)) {
3370 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_SINT node\n");
3375 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG))
3379 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG)) {
3381 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node,1), Tmp2);
3384 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_UINT node\n");
3390 Results.push_back(TLI.expandFP_TO_INT_SAT(
Node, DAG));
3400 if (
Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3403 Node->getOperand(0));
3405 Tmp1 = ExpandExtractFromVectorThroughStack(
SDValue(
Node, 0));
3415 Results.push_back(ExpandVectorBuildThroughStack(
Node));
3421 Results.push_back(ExpandINSERT_VECTOR_ELT(
Node->getOperand(0),
3422 Node->getOperand(1),
3423 Node->getOperand(2), dl));
3429 EVT VT =
Node->getValueType(0);
3433 if (!TLI.isTypeLegal(EltVT)) {
3434 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3439 if (NewEltVT.
bitsLT(EltVT)) {
3455 unsigned int factor =
3463 for (
unsigned fi = 0; fi < factor; ++fi)
3467 for (
unsigned fi = 0; fi < factor; ++fi)
3478 for (
unsigned i = 0; i != NumElems; ++i) {
3486 DAG.getVectorIdxConstant(
Idx, dl)));
3490 DAG.getVectorIdxConstant(
Idx - NumElems, dl)));
3493 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3500 Results.push_back(TLI.expandVectorSplice(
Node, DAG));
3504 EVT OpTy =
Node->getOperand(0).getValueType();
3505 if (
Node->getConstantOperandVal(1)) {
3509 TLI.getShiftAmountTy(
3510 Node->getOperand(0).getValueType(),
3511 DAG.getDataLayout())));
3516 Node->getOperand(0));
3524 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3525 Results.push_back(DAG.getCopyFromReg(
Node->getOperand(0), dl, SP,
3526 Node->getValueType(0)));
3529 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
3536 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3537 Results.push_back(DAG.getCopyToReg(
Node->getOperand(0), dl, SP,
3538 Node->getOperand(1)));
3544 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3559 TLI.expandIS_FPCLASS(
Node->getValueType(0),
Node->getOperand(0),
3570 switch (
Node->getOpcode()) {
3577 Tmp1 =
Node->getOperand(0);
3578 Tmp2 =
Node->getOperand(1);
3579 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3585 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG))
3591 EVT VT =
Node->getValueType(0);
3597 SDVTList VTs = DAG.getVTList(VT, VT);
3607 EVT VT =
Node->getValueType(0);
3611 if (TLI.getLibcallName(LC))
3617 Results.push_back(Expanded.getValue(1));
3626 if (TLI.getLibcallName(LC))
3631 Results.push_back(Expanded.getValue(1));
3639 if (
Node->getValueType(0) != MVT::f32) {
3651 if (
Node->getValueType(0) != MVT::f32) {
3656 {Node->getOperand(0), Node->getOperand(1)});
3658 {
Node->getValueType(0), MVT::Other},
3666 if (!TLI.useSoftFloat() &&
TM.Options.UnsafeFPMath) {
3668 MVT SVT =
Op.getSimpleValueType();
3669 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3675 DAG.getIntPtrConstant(0, dl,
true));
3686 DAG.shouldOptForSize()))
3687 Results.push_back(ExpandConstantFP(CFP,
true));
3692 Results.push_back(ExpandConstant(CP));
3696 EVT VT =
Node->getValueType(0);
3697 if (TLI.isOperationLegalOrCustom(
ISD::FADD, VT) &&
3698 TLI.isOperationLegalOrCustom(
ISD::FNEG, VT)) {
3707 EVT VT =
Node->getValueType(0);
3709 TLI.isOperationLegalOrCustom(
ISD::XOR, VT) &&
3710 "Don't know how to expand this subtraction!");
3711 Tmp1 = DAG.getNOT(dl,
Node->getOperand(1), VT);
3712 Tmp1 = DAG.
getNode(
ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3718 if (TLI.expandREM(
Node, Tmp1, DAG))
3725 EVT VT =
Node->getValueType(0);
3726 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3727 SDVTList VTs = DAG.getVTList(VT, VT);
3728 Tmp1 = DAG.
getNode(DivRemOpc, dl, VTs,
Node->getOperand(0),
3729 Node->getOperand(1));
3736 unsigned ExpandOpcode =
3738 EVT VT =
Node->getValueType(0);
3739 SDVTList VTs = DAG.getVTList(VT, VT);
3741 Tmp1 = DAG.
getNode(ExpandOpcode, dl, VTs,
Node->getOperand(0),
3742 Node->getOperand(1));
3750 MVT VT =
LHS.getSimpleValueType();
3751 unsigned MULHOpcode =
3754 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3756 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3762 assert(TLI.isTypeLegal(HalfType));
3763 if (TLI.expandMUL_LOHI(
Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3765 TargetLowering::MulExpansionKind::Always)) {
3766 for (
unsigned i = 0; i < 2; ++i) {
3769 SDValue Shift = DAG.getConstant(
3771 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3780 EVT VT =
Node->getValueType(0);
3781 SDVTList VTs = DAG.getVTList(VT, VT);
3787 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::SMUL_LOHI, VT);
3788 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::UMUL_LOHI, VT);
3789 bool HasMULHS = TLI.isOperationLegalOrCustom(
ISD::MULHS, VT);
3790 bool HasMULHU = TLI.isOperationLegalOrCustom(
ISD::MULHU, VT);
3791 unsigned OpToUse = 0;
3792 if (HasSMUL_LOHI && !HasMULHS) {
3794 }
else if (HasUMUL_LOHI && !HasMULHU) {
3796 }
else if (HasSMUL_LOHI) {
3798 }
else if (HasUMUL_LOHI) {
3802 Results.push_back(DAG.getNode(OpToUse, dl, VTs,
Node->getOperand(0),
3803 Node->getOperand(1)));
3811 TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
3812 TLI.isOperationLegalOrCustom(
ISD::OR, VT) &&
3813 TLI.expandMUL(
Node,
Lo,
Hi, HalfType, DAG,
3814 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3819 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3827 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG))
3832 if (
SDValue Expanded = TLI.expandROT(
Node,
true , DAG))
3839 Results.push_back(TLI.expandAddSubSat(
Node, DAG));
3849 Results.push_back(TLI.expandFixedPointMul(
Node, DAG));
3856 Node->getOperand(0),
3857 Node->getOperand(1),
3858 Node->getConstantOperandVal(2),
3881 EVT VT =
LHS.getValueType();
3885 EVT CarryType =
Node->getValueType(1);
3886 EVT SetCCType = getSetCCResultType(
Node->getValueType(0));
3888 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS,
CC);
3891 SDValue One = DAG.getConstant(1, dl, VT);
3893 DAG.
getNode(
ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3902 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero,
ISD::SETEQ)
3903 : DAG.getSetCC(dl, SetCCType, Sum, Zero,
ISD::SETEQ);
3905 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3911 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3917 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
3925 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
3933 if (TLI.expandMULO(
Node, Result, Overflow, DAG)) {
3945 DAG.getConstant(
PairTy.getSizeInBits() / 2, dl,
3946 TLI.getShiftAmountTy(
PairTy, DAG.getDataLayout())));
3951 Tmp1 =
Node->getOperand(0);
3952 Tmp2 =
Node->getOperand(1);
3953 Tmp3 =
Node->getOperand(2);
3957 cast<CondCodeSDNode>(Tmp1.
getOperand(2))->get());
3959 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3970 int JTI = cast<JumpTableSDNode>(Table.
getNode())->getIndex();
3973 EVT PTy = TLI.getPointerTy(TD);
3975 unsigned EntrySize =
3976 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3983 Index = DAG.getNode(
3988 DAG.getConstant(EntrySize, dl,
Index.getValueType()));
3997 if (TLI.isJumpTableRelative()) {
4002 TLI.getPICJumpTableRelocBase(Table, DAG));
4005 Tmp1 = TLI.expandIndirectJTBranch(dl,
LD.getValue(1),
Addr, JTI, DAG);
4012 Tmp1 =
Node->getOperand(0);
4013 Tmp2 =
Node->getOperand(1);
4019 Node->getOperand(2));
4031 Node->getOperand(2));
4039 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
4044 unsigned Offset = IsStrict ? 1 : 0;
4053 bool Legalized = TLI.LegalizeSetCCCondCode(
4054 DAG,
Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4055 Chain, IsSignaling);
4063 {Chain, Tmp1, Tmp2, Tmp3},
Node->getFlags());
4067 {Tmp1, Tmp2, Tmp3, Mask, EVL},
Node->getFlags());
4069 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl,
Node->getValueType(0), Tmp1,
4070 Tmp2, Tmp3,
Node->getFlags());
4078 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->
getValueType(0));
4081 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->
getValueType(0));
4093 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
4098 EVT VT =
Node->getValueType(0);
4101 DAG.getBoolConstant(
true, dl, VT, Tmp1VT),
4102 DAG.getBoolConstant(
false, dl, VT, Tmp1VT), Tmp3);
4109 Tmp1 =
Node->getOperand(0);
4110 Tmp2 =
Node->getOperand(1);
4111 Tmp3 =
Node->getOperand(2);
4112 Tmp4 =
Node->getOperand(3);
4113 EVT VT =
Node->getValueType(0);
4123 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4125 EVT CCVT = getSetCCResultType(CmpVT);
4127 Results.push_back(DAG.getSelect(dl, VT,
Cond, Tmp3, Tmp4));
4132 bool Legalized =
false;
4140 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4150 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4156 Legalized = TLI.LegalizeSetCCCondCode(
4160 assert(Legalized &&
"Can't legalize SELECT_CC with legal condition!");
4171 Tmp1, Tmp2, Tmp3, Tmp4,
CC);
4176 Tmp2, Tmp3, Tmp4,
CC);
4186 Tmp1 =
Node->getOperand(0);
4187 Tmp2 =
Node->getOperand(2);
4188 Tmp3 =
Node->getOperand(3);
4189 Tmp4 =
Node->getOperand(1);
4191 bool Legalized = TLI.LegalizeSetCCCondCode(
4192 DAG, getSetCCResultType(Tmp2.
getValueType()), Tmp2, Tmp3, Tmp4,
4195 assert(Legalized &&
"Can't legalize BR_CC with legal condition!");
4200 assert(!NeedInvert &&
"Don't know how to invert BR_CC!");
4203 Tmp4, Tmp2, Tmp3,
Node->getOperand(4));
4208 Tmp2, Tmp3,
Node->getOperand(4));
4223 EVT VT =
Node->getValueType(0);
4229 for (
unsigned Idx = 0;
Idx < NumElem;
Idx++) {
4232 Node->getOperand(0), DAG.getVectorIdxConstant(
Idx, dl));
4235 Node->getOperand(1), DAG.getVectorIdxConstant(
Idx, dl));
4259 Results.push_back(TLI.expandVecReduce(
Node, DAG));
4276 if (!TLI.isStrictFPEnabled() &&
Results.
empty() &&
Node->isStrictFPOpcode()) {
4282 switch (
Node->getOpcode()) {
4284 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4285 Node->getValueType(0))
4286 == TargetLowering::Legal)
4290 if (TLI.getStrictFPOperationAction(
4293 if (TLI.getStrictFPOperationAction(
4297 EVT VT =
Node->getValueType(0);
4301 {Node->getOperand(0), Node->getOperand(1), Neg},
4316 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4317 Node->getOperand(1).getValueType())
4318 == TargetLowering::Legal)
4335void SelectionDAGLegalize::ConvertNodeToLibcall(
SDNode *
Node) {
4340 unsigned Opc =
Node->getOpcode();
4349 .setChain(
Node->getOperand(0))
4352 DAG.getExternalSymbol(
"__sync_synchronize",
4353 TLI.getPointerTy(DAG.getDataLayout())),
4356 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4358 Results.push_back(CallResult.second);
4380 EVT RetVT =
Node->getValueType(0);
4383 if (TLI.getLibcallName(LC)) {
4390 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4391 "Unexpected atomic op or value type!");
4395 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4398 Node->getOperand(0));
4400 Results.push_back(Tmp.second);
4408 .setChain(
Node->getOperand(0))
4410 DAG.getExternalSymbol(
4411 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4413 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4415 Results.push_back(CallResult.second);
4420 ExpandFPLibCall(
Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4421 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4422 RTLIB::FMIN_PPCF128,
Results);
4429 ExpandFPLibCall(
Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4430 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4431 RTLIB::FMAX_PPCF128,
Results);
4435 ExpandFPLibCall(
Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4436 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4437 RTLIB::SQRT_PPCF128,
Results);
4440 ExpandFPLibCall(
Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4441 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4442 RTLIB::CBRT_PPCF128,
Results);
4446 ExpandFPLibCall(
Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4447 RTLIB::SIN_F80, RTLIB::SIN_F128,
4452 ExpandFPLibCall(
Node, RTLIB::COS_F32, RTLIB::COS_F64,
4453 RTLIB::COS_F80, RTLIB::COS_F128,
4462 ExpandFPLibCall(
Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4463 RTLIB::LOG_F128, RTLIB::LOG_PPCF128,
Results);
4467 ExpandFPLibCall(
Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4468 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128,
Results);
4472 ExpandFPLibCall(
Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4473 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128,
Results);
4477 ExpandFPLibCall(
Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4478 RTLIB::EXP_F128, RTLIB::EXP_PPCF128,
Results);
4482 ExpandFPLibCall(
Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4483 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128,
Results);
4486 ExpandFPLibCall(
Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4487 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128,
Results);
4491 ExpandFPLibCall(
Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4492 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4493 RTLIB::TRUNC_PPCF128,
Results);
4497 ExpandFPLibCall(
Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4498 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4499 RTLIB::FLOOR_PPCF128,
Results);
4503 ExpandFPLibCall(
Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4504 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4505 RTLIB::CEIL_PPCF128,
Results);
4509 ExpandFPLibCall(
Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4510 RTLIB::RINT_F80, RTLIB::RINT_F128,
4511 RTLIB::RINT_PPCF128,
Results);
4515 ExpandFPLibCall(
Node, RTLIB::NEARBYINT_F32,
4516 RTLIB::NEARBYINT_F64,
4517 RTLIB::NEARBYINT_F80,
4518 RTLIB::NEARBYINT_F128,
4519 RTLIB::NEARBYINT_PPCF128,
Results);
4523 ExpandFPLibCall(
Node, RTLIB::ROUND_F32,
4527 RTLIB::ROUND_PPCF128,
Results);
4531 ExpandFPLibCall(
Node, RTLIB::ROUNDEVEN_F32,
4532 RTLIB::ROUNDEVEN_F64,
4533 RTLIB::ROUNDEVEN_F80,
4534 RTLIB::ROUNDEVEN_F128,
4535 RTLIB::ROUNDEVEN_PPCF128,
Results);
4539 ExpandFPLibCall(
Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4540 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128,
Results);
4549 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
4550 if (!TLI.getLibcallName(LC)) {
4552 if (
Node->isStrictFPOpcode()) {
4555 {
Node->getValueType(0),
Node->getValueType(1)},
4556 {
Node->getOperand(0),
Node->getOperand(2)});
4559 {
Node->getValueType(0),
Node->getValueType(1)},
4566 Node->getOperand(1));
4568 Node->getValueType(0),
4573 unsigned Offset =
Node->isStrictFPOpcode() ? 1 : 0;
4574 bool ExponentHasSizeOfInt =
4575 DAG.getLibInfo().getIntSize() ==
4576 Node->getOperand(1 +
Offset).getValueType().getSizeInBits();
4577 if (!ExponentHasSizeOfInt) {
4580 DAG.getContext()->emitError(
"POWI exponent does not match sizeof(int)");
4581 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
4589 ExpandFPLibCall(
Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4590 RTLIB::POW_F128, RTLIB::POW_PPCF128,
Results);
4594 ExpandArgFPLibCall(
Node, RTLIB::LROUND_F32,
4595 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4597 RTLIB::LROUND_PPCF128,
Results);
4601 ExpandArgFPLibCall(
Node, RTLIB::LLROUND_F32,
4602 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4603 RTLIB::LLROUND_F128,
4604 RTLIB::LLROUND_PPCF128,
Results);
4608 ExpandArgFPLibCall(
Node, RTLIB::LRINT_F32,
4609 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4611 RTLIB::LRINT_PPCF128,
Results);
4615 ExpandArgFPLibCall(
Node, RTLIB::LLRINT_F32,
4616 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4618 RTLIB::LLRINT_PPCF128,
Results);
4622 ExpandFPLibCall(
Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4623 RTLIB::DIV_F80, RTLIB::DIV_F128,
4628 ExpandFPLibCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
4629 RTLIB::REM_F80, RTLIB::REM_F128,
4634 ExpandFPLibCall(
Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4635 RTLIB::FMA_F80, RTLIB::FMA_F128,
4640 ExpandFPLibCall(
Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4641 RTLIB::ADD_F80, RTLIB::ADD_F128,
4646 ExpandFPLibCall(
Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4647 RTLIB::MUL_F80, RTLIB::MUL_F128,
4651 if (
Node->getValueType(0) == MVT::f32) {
4652 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32,
Node,
false).first);
4656 if (
Node->getValueType(0) == MVT::f32) {
4658 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4659 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32,
Node->getOperand(1),
4662 Results.push_back(Tmp.second);
4666 if (
Node->getValueType(0) == MVT::f32) {
4668 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4669 DAG, RTLIB::FPEXT_F16_F32, MVT::f32,
Node->getOperand(1), CallOptions,
4672 Results.push_back(Tmp.second);
4679 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16");
4680 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4686 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_bf16");
4687 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4695 bool IsStrict =
Node->isStrictFPOpcode();
4698 EVT SVT =
Node->getOperand(IsStrict ? 1 : 0).getValueType();
4699 EVT RVT =
Node->getValueType(0);
4707 for (
unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4708 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4716 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4721 NVT,
Node->getOperand(IsStrict ? 1 : 0));
4724 std::pair<SDValue, SDValue> Tmp =
4725 TLI.makeLibCall(DAG, LC, RVT,
Op, CallOptions, dl, Chain);
4728 Results.push_back(Tmp.second);
4736 bool IsStrict =
Node->isStrictFPOpcode();
4741 EVT SVT =
Op.getValueType();
4742 EVT RVT =
Node->getValueType(0);
4750 for (
unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4751 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4759 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4763 std::pair<SDValue, SDValue> Tmp =
4764 TLI.makeLibCall(DAG, LC, NVT,
Op, CallOptions, dl, Chain);
4769 Results.push_back(Tmp.second);
4780 bool IsStrict =
Node->isStrictFPOpcode();
4783 EVT VT =
Node->getValueType(0);
4784 assert(cast<ConstantSDNode>(
Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4785 "Unable to expand as libcall if it is not normal rounding");
4788 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4791 std::pair<SDValue, SDValue> Tmp =
4792 TLI.makeLibCall(DAG, LC, VT,
Op, CallOptions,
SDLoc(
Node), Chain);
4795 Results.push_back(Tmp.second);
4801 Node->getValueType(0)),
4802 Node,
false).first);
4815 Node->getValueType(0));
4817 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4820 std::pair<SDValue, SDValue> Tmp =
4821 TLI.makeLibCall(DAG, LC,
Node->getValueType(0),
Node->getOperand(1),
4824 Results.push_back(Tmp.second);
4829 ExpandFPLibCall(
Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4830 RTLIB::SUB_F80, RTLIB::SUB_F128,
4836 RTLIB::SREM_I16, RTLIB::SREM_I32,
4837 RTLIB::SREM_I64, RTLIB::SREM_I128));
4842 RTLIB::UREM_I16, RTLIB::UREM_I32,
4843 RTLIB::UREM_I64, RTLIB::UREM_I128));
4848 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4849 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4854 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4855 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4865 RTLIB::MUL_I16, RTLIB::MUL_I32,
4866 RTLIB::MUL_I64, RTLIB::MUL_I128));
4869 switch (
Node->getSimpleValueType(0).SimpleTy) {
4873 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32,
Node,
false).first);
4876 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64,
Node,
false).first);
4879 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128,
Node,
false).first);
4886 SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
4889 DAG.makeStateFunctionCall(RTLIB::FESETENV,
Ptr, Chain, dl));
4896 DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl));
4903 DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl));
4909 EVT ModeVT =
Node->getValueType(0);
4911 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4912 SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr,
4913 Node->getOperand(0), dl);
4915 ModeVT, dl, Chain, StackPtr,
4925 EVT ModeVT =
Mode.getValueType();
4927 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4928 SDValue StInst = DAG.getStore(
4929 Node->getOperand(0), dl, Mode, StackPtr,
4932 DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl));
4940 EVT PtrTy = TLI.getPointerTy(
DL);
4942 Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
4943 Node->getOperand(0), dl));
4950 LLVM_DEBUG(
dbgs() <<
"Successfully converted node to libcall\n");
4959 MVT EltVT,
MVT NewEltVT) {
4961 MVT MidVT = OldEltsPerNewElt == 1
4968void SelectionDAGLegalize::PromoteNode(
SDNode *
Node) {
4971 MVT OVT =
Node->getSimpleValueType(0);
4977 OVT =
Node->getOperand(0).getSimpleValueType();
4983 OVT =
Node->getOperand(1).getSimpleValueType();
4986 OVT =
Node->getOperand(2).getSimpleValueType();
4987 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), OVT);
4989 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4990 switch (
Node->getOpcode()) {
5010 DAG.getConstant(TopBit, dl, NVT));
5014 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5028 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5031 DAG.getConstant(DiffBits, dl,
5032 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
5045 Results.push_back(PromoteLegalFP_TO_INT_SAT(
Node, dl));
5062 &&
"VAARG promotion is supported only for vectors or integer types");
5067 Tmp1 = DAG.getVAArg(NVT, dl, Chain,
Ptr,
Node->getOperand(2),
5068 Node->getConstantOperandVal(3));
5071 Tmp2 = DAG.
getNode(TruncOp, dl, OVT, Tmp1);
5075 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), Tmp2);
5076 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
5078 UpdatedNodes->insert(Tmp2.
getNode());
5079 UpdatedNodes->insert(Chain.
getNode());
5096 unsigned ExtOp, TruncOp;
5103 switch (
Node->getOpcode()) {
5119 if (TLI.isSExtCheaperThanZExt(OVT, NVT))
5128 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5129 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5131 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5132 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
5140 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5141 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5144 auto &
DL = DAG.getDataLayout();
5148 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(
DL, NVT)));
5154 unsigned ExtOp, TruncOp;
5155 if (
Node->getValueType(0).isVector() ||
5159 }
else if (
Node->getValueType(0).isInteger()) {
5166 Tmp1 =
Node->getOperand(0);
5168 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5169 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5171 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5174 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1);
5176 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1,
5177 DAG.getIntPtrConstant(0, dl));
5189 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5198 Node->getOperand(2));
5206 MVT CVT =
Node->getSimpleValueType(0);
5207 assert(CVT == OVT &&
"not handled");
5215 if (TLI.isCondCodeLegal(CCCode, CVT)) {
5216 Tmp1 =
Node->getOperand(0);
5217 Tmp2 =
Node->getOperand(1);
5219 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5220 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5223 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5224 Tmp4 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5234 DAG.getIntPtrConstant(0, dl,
true));
5246 TLI.isSExtCheaperThanZExt(
Node->getOperand(0).getValueType(), NVT))
5251 if (
Node->isStrictFPOpcode()) {
5253 std::tie(Tmp1, std::ignore) =
5254 DAG.getStrictFPExtendOrRound(
Node->getOperand(1), InChain, dl, NVT);
5255 std::tie(Tmp2, std::ignore) =
5256 DAG.getStrictFPExtendOrRound(
Node->getOperand(2), InChain, dl, NVT);
5258 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
5259 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
5260 Results.push_back(DAG.getNode(
Node->getOpcode(), dl, VTs,
5261 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5266 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5267 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5269 Tmp2,
Node->getOperand(2),
Node->getFlags()));
5276 cast<CondCodeSDNode>(
Node->getOperand(1))->get();
5279 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5280 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5282 Node->getOperand(0),
Node->getOperand(1),
5283 Tmp1, Tmp2,
Node->getOperand(4)));
5298 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
5302 DAG.getIntPtrConstant(0, dl,
true)));
5313 {
Node->getOperand(0),
Node->getOperand(1)});
5315 {
Node->getOperand(0),
Node->getOperand(2)});
5318 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5319 {Tmp3, Tmp1, Tmp2});
5321 {Tmp1.
getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5331 DAG.getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5332 DAG.getIntPtrConstant(0, dl,
true)));
5336 {
Node->getOperand(0),
Node->getOperand(1)});
5338 {
Node->getOperand(0),
Node->getOperand(2)});
5340 {
Node->getOperand(0),
Node->getOperand(3)});
5343 Tmp4 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5344 {Tmp4, Tmp1, Tmp2, Tmp3});
5346 {Tmp4.
getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5354 Tmp2 =
Node->getOperand(1);
5355 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5365 DAG.getIntPtrConstant(isTrunc, dl,
true)));
5370 {
Node->getOperand(0),
Node->getOperand(1)});
5371 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5374 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5384 DAG.getIntPtrConstant(0, dl,
true)));
5409 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5412 DAG.getIntPtrConstant(0, dl,
true)));
5430 {
Node->getOperand(0),
Node->getOperand(1)});
5431 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5434 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5449 "Invalid promote type for build_vector");
5455 for (
unsigned I = 0,
E =
Node->getNumOperands();
I !=
E; ++
I) {
5484 "Invalid promote type for extract_vector_elt");
5491 EVT IdxVT =
Idx.getValueType();
5493 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
5499 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5500 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5508 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
5530 "Invalid promote type for insert_vector_elt");
5538 EVT IdxVT =
Idx.getValueType();
5541 SDValue Factor = DAG.getConstant(NewEltsPerOldElt,
SDLoc(), IdxVT);
5548 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5549 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5553 CastVal, IdxOffset);
5556 NewVec, Elt, InEltIdx);
5595 "unexpected promotion type");
5597 "unexpected atomic_swap with illegal type");
5601 DAG.getVTList(NVT, MVT::Other),
5602 { AM->getChain(), AM->getBasePtr(), CastVal },
5610 MVT ScalarType =
Scalar.getSimpleValueType();
5614 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5619 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5622 DAG.getIntPtrConstant(0, dl,
true)));
5648 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes);
5655 bool AnyLegalized =
false;
5666 if (LegalizedNodes.
insert(
N).second) {
5667 AnyLegalized =
true;
5688 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes, &UpdatedNodes);
5695 return LegalizedNodes.
count(
N);
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
static MaybeAlign getAlign(Value *Ptr)
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI)
Return true if sincos libcall is available.
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
static MachineMemOperand * getStackAlignedMMO(SDValue StackPtr, MachineFunction &MF, bool isObjectScalable)
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This is an SDNode representing atomic operations.
const SDValue & getVal() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
const BasicBlock & back() const
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOStore
The memory access writes data.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetLowering & getTargetLoweringInfo() const
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
allnodes_const_iterator allnodes_end() const
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const DataLayout & getDataLayout() const
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
LLVMContext * getContext() const
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
Information about stack frame layout on the target.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
constexpr ScalarTy getFixedValue() const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ FRAME_TO_ARGS_OFFSET
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
@ RESET_FPENV
Set floating-point environment to default state.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ GLOBAL_OFFSET_TABLE
The address of the GOT.
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ SET_FPENV_MEM
Sets the current floating point environment.
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
@ Undef
Value of the register doesn't matter.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
int32_t ExponentType
A signed type to represent a floating point numbers unbiased exponent.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)