58#define DEBUG_TYPE "legalizedag"
64struct FloatSignAsInt {
87class SelectionDAGLegalize {
99 EVT getSetCCResultType(
EVT VT)
const {
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
166 void getSignAsIntValue(FloatSignAsInt &State,
const SDLoc &
DL,
168 SDValue modifySignAsInt(
const FloatSignAsInt &State,
const SDLoc &
DL,
208 dbgs() <<
" with: "; New->dump(&DAG));
211 "Replacing one node with another that produces a different number "
215 UpdatedNodes->
insert(New);
221 dbgs() <<
" with: "; New->dump(&DAG));
225 UpdatedNodes->
insert(New.getNode());
226 ReplacedNode(Old.getNode());
233 for (
unsigned i = 0, e = Old->
getNumValues(); i != e; ++i) {
237 UpdatedNodes->
insert(New[i].getNode());
244 dbgs() <<
" with: "; New->dump(&DAG));
248 UpdatedNodes->
insert(New.getNode());
249 ReplacedNode(Old.getNode());
259 bool isObjectScalable) {
261 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();
267 ObjectSize, MFI.getObjectAlign(FI));
274SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
279 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
281 assert(NumEltsGrowth &&
"Cannot promote to vector type with fewer elts!");
283 if (NumEltsGrowth == 1)
284 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
287 for (
unsigned i = 0; i != NumMaskElts; ++i) {
289 for (
unsigned j = 0;
j != NumEltsGrowth; ++
j) {
296 assert(NewMask.
size() == NumDestElts &&
"Non-integer NumEltsGrowth?");
297 assert(TLI.isShuffleMaskLegal(NewMask, NVT) &&
"Shuffle not legal?");
298 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
317 assert((VT == MVT::f64 || VT == MVT::f32) &&
"Invalid type expansion");
319 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
329 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
335 TLI.ShouldShrinkFPConstant(OrigVT)) {
338 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
346 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
347 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
356 OrigVT, dl, DAG.getEntryNode(), CPIdx,
364 EVT VT =
CP->getValueType(0);
365 SDValue CPIdx = DAG.getConstantPool(
CP->getConstantIntValue(),
366 TLI.getPointerTy(DAG.getDataLayout()));
367 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
369 VT, dl, DAG.getEntryNode(), CPIdx,
395 for (
unsigned i = 0; i != NumElts; ++i)
396 ShufOps.
push_back(i != InsertPos->getZExtValue() ? i : NumElts);
398 return DAG.getVectorShuffle(Vec.
getValueType(), dl, Vec, ScVec, ShufOps);
401 return ExpandInsertToVectorThroughStack(
Op);
428 TLI.isTypeLegal(MVT::i32)) {
430 bitcastToAPInt().zextOrTrunc(32),
431 SDLoc(CFP), MVT::i32);
432 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
433 ST->getOriginalAlign(), MMOFlags, AAInfo);
439 if (TLI.isTypeLegal(MVT::i64)) {
441 zextOrTrunc(64),
SDLoc(CFP), MVT::i64);
442 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
443 ST->getOriginalAlign(), MMOFlags, AAInfo);
446 if (TLI.isTypeLegal(MVT::i32) && !
ST->isVolatile()) {
453 if (DAG.getDataLayout().isBigEndian())
456 Lo = DAG.getStore(Chain, dl,
Lo,
Ptr,
ST->getPointerInfo(),
457 ST->getOriginalAlign(), MMOFlags, AAInfo);
459 Hi = DAG.getStore(Chain, dl,
Hi,
Ptr,
460 ST->getPointerInfo().getWithOffset(4),
461 ST->getOriginalAlign(), MMOFlags, AAInfo);
470void SelectionDAGLegalize::LegalizeStoreOps(
SDNode *
Node) {
479 if (!
ST->isTruncatingStore()) {
481 if (
SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
482 ReplaceNode(ST, OptStore);
487 MVT VT =
Value.getSimpleValueType();
488 switch (TLI.getOperationAction(
ISD::STORE, VT)) {
490 case TargetLowering::Legal: {
493 EVT MemVT =
ST->getMemoryVT();
495 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
496 *
ST->getMemOperand())) {
499 ReplaceNode(
SDValue(ST, 0), Result);
504 case TargetLowering::Custom: {
511 case TargetLowering::Promote: {
514 "Can only promote stores to same size type");
517 ST->getOriginalAlign(), MMOFlags, AAInfo);
527 EVT StVT =
ST->getMemoryVT();
530 auto &
DL = DAG.getDataLayout();
532 if (StWidth != StSize) {
537 Value = DAG.getZeroExtendInReg(
Value, dl, StVT);
539 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), NVT,
540 ST->getOriginalAlign(), MMOFlags, AAInfo);
546 unsigned LogStWidth =
Log2_32(StWidthBits);
548 unsigned RoundWidth = 1 << LogStWidth;
549 assert(RoundWidth < StWidthBits);
550 unsigned ExtraWidth = StWidthBits - RoundWidth;
551 assert(ExtraWidth < RoundWidth);
552 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
553 "Store size not an integral number of bytes!");
557 unsigned IncrementSize;
559 if (
DL.isLittleEndian()) {
562 Lo = DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(),
563 RoundVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
566 IncrementSize = RoundWidth / 8;
571 DAG.getConstant(RoundWidth, dl,
572 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
573 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
574 ST->getPointerInfo().getWithOffset(IncrementSize),
575 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
582 DAG.getConstant(ExtraWidth, dl,
583 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
584 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
ST->getPointerInfo(), RoundVT,
585 ST->getOriginalAlign(), MMOFlags, AAInfo);
588 IncrementSize = RoundWidth / 8;
590 DAG.getConstant(IncrementSize, dl,
591 Ptr.getValueType()));
593 ST->getPointerInfo().getWithOffset(IncrementSize),
594 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
601 switch (TLI.getTruncStoreAction(
ST->getValue().getValueType(), StVT)) {
603 case TargetLowering::Legal: {
604 EVT MemVT =
ST->getMemoryVT();
607 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
608 *
ST->getMemOperand())) {
610 ReplaceNode(
SDValue(ST, 0), Result);
614 case TargetLowering::Custom: {
620 case TargetLowering::Expand:
622 "Vector Stores are handled in LegalizeVectorOps");
627 if (TLI.isTypeLegal(StVT)) {
630 ST->getOriginalAlign(), MMOFlags, AAInfo);
635 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
638 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), StVT,
639 ST->getOriginalAlign(), MMOFlags, AAInfo);
648void SelectionDAGLegalize::LegalizeLoadOps(
SDNode *
Node) {
657 LLVM_DEBUG(
dbgs() <<
"Legalizing non-extending load operation\n");
658 MVT VT =
Node->getSimpleValueType(0);
662 switch (TLI.getOperationAction(
Node->getOpcode(), VT)) {
664 case TargetLowering::Legal: {
665 EVT MemVT =
LD->getMemoryVT();
669 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
670 *
LD->getMemOperand())) {
671 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
675 case TargetLowering::Custom:
676 if (
SDValue Res = TLI.LowerOperation(RVal, DAG)) {
682 case TargetLowering::Promote: {
683 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
685 "Can only promote loads to same size type");
687 SDValue Res = DAG.getLoad(NVT, dl, Chain,
Ptr,
LD->getMemOperand());
695 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), RVal);
696 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), RChain);
698 UpdatedNodes->insert(RVal.
getNode());
699 UpdatedNodes->insert(RChain.
getNode());
707 EVT SrcVT =
LD->getMemoryVT();
721 TLI.getLoadExtAction(ExtType,
Node->getValueType(0), MVT::i1) ==
722 TargetLowering::Promote)) {
736 Chain,
Ptr,
LD->getPointerInfo(), NVT,
737 LD->getOriginalAlign(), MMOFlags, AAInfo);
745 Result, DAG.getValueType(SrcVT));
749 Result.getValueType(), Result,
750 DAG.getValueType(SrcVT));
758 unsigned LogSrcWidth =
Log2_32(SrcWidthBits);
760 unsigned RoundWidth = 1 << LogSrcWidth;
761 assert(RoundWidth < SrcWidthBits);
762 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
763 assert(ExtraWidth < RoundWidth);
764 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
765 "Load size not an integral number of bytes!");
769 unsigned IncrementSize;
770 auto &
DL = DAG.getDataLayout();
772 if (
DL.isLittleEndian()) {
776 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
780 IncrementSize = RoundWidth / 8;
783 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
784 LD->getPointerInfo().getWithOffset(IncrementSize),
785 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
795 DAG.getConstant(RoundWidth, dl,
796 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
804 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
805 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
809 IncrementSize = RoundWidth / 8;
813 LD->getPointerInfo().getWithOffset(IncrementSize),
814 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
824 DAG.getConstant(ExtraWidth, dl,
825 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
833 bool isCustom =
false;
834 switch (TLI.getLoadExtAction(ExtType,
Node->getValueType(0),
837 case TargetLowering::Custom:
840 case TargetLowering::Legal:
852 EVT MemVT =
LD->getMemoryVT();
854 if (!TLI.allowsMemoryAccess(*DAG.getContext(),
DL, MemVT,
855 *
LD->getMemOperand())) {
856 std::tie(
Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
861 case TargetLowering::Expand: {
862 EVT DestVT =
Node->getValueType(0);
868 (TLI.isTypeLegal(SrcVT) ||
869 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
876 SrcVT,
LD->getMemOperand());
879 Value = DAG.getNode(ExtendOp, dl,
Node->getValueType(0), Load);
880 Chain =
Load.getValue(1);
889 if (SVT == MVT::f16 || SVT == MVT::bf16) {
895 Ptr, ISrcVT,
LD->getMemOperand());
899 Chain =
Result.getValue(1);
905 "Vector Loads are handled in LegalizeVectorOps");
912 "EXTLOAD should always be supported!");
916 Node->getValueType(0),
918 LD->getMemOperand());
923 Result, DAG.getValueType(SrcVT));
925 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
927 Chain =
Result.getValue(1);
938 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
940 UpdatedNodes->insert(
Value.getNode());
941 UpdatedNodes->insert(Chain.
getNode());
948void SelectionDAGLegalize::LegalizeOp(
SDNode *
Node) {
957 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
958 assert(TLI.getTypeAction(*DAG.getContext(),
Node->getValueType(i)) ==
959 TargetLowering::TypeLegal &&
960 "Unexpected illegal type!");
963 assert((TLI.getTypeAction(*DAG.getContext(),
Op.getValueType()) ==
964 TargetLowering::TypeLegal ||
967 "Unexpected illegal type!");
972 bool SimpleFinishLegalizing =
true;
973 switch (
Node->getOpcode()) {
978 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
981 Action = TLI.getOperationAction(
Node->getOpcode(),
982 Node->getValueType(0));
985 Action = TLI.getOperationAction(
Node->getOpcode(),
986 Node->getValueType(0));
987 if (Action != TargetLowering::Promote)
988 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
992 Action = TLI.getOperationAction(
Node->getOpcode(),
993 Node->getOperand(1).getValueType());
1004 Action = TLI.getOperationAction(
Node->getOpcode(),
1005 Node->getOperand(0).getValueType());
1018 Action = TLI.getOperationAction(
Node->getOpcode(),
1019 Node->getOperand(1).getValueType());
1022 EVT InnerType = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1023 Action = TLI.getOperationAction(
Node->getOpcode(), InnerType);
1027 Action = TLI.getOperationAction(
Node->getOpcode(),
1028 Node->getOperand(1).getValueType());
1037 unsigned Opc =
Node->getOpcode();
1042 : (Opc ==
ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1044 unsigned CompareOperand = Opc ==
ISD::BR_CC ? 2
1048 MVT OpVT =
Node->getOperand(CompareOperand).getSimpleValueType();
1050 cast<CondCodeSDNode>(
Node->getOperand(CCOperand))->get();
1051 Action = TLI.getCondCodeAction(CCCode, OpVT);
1052 if (Action == TargetLowering::Legal) {
1054 Action = TLI.getOperationAction(
Node->getOpcode(),
1055 Node->getValueType(0));
1057 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
1065 SimpleFinishLegalizing =
false;
1072 SimpleFinishLegalizing =
false;
1085 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1086 if (Action == TargetLowering::Legal)
1087 Action = TargetLowering::Expand;
1097 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1098 if (Action == TargetLowering::Legal)
1099 Action = TargetLowering::Custom;
1105 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::i64);
1112 Action = TargetLowering::Legal;
1115 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1116 if (Action == TargetLowering::Expand) {
1120 Node->getOperand(0));
1127 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1128 if (Action == TargetLowering::Expand) {
1132 Node->getOperand(0));
1146 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1156 unsigned Scale =
Node->getConstantOperandVal(2);
1157 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
1158 Node->getValueType(0), Scale);
1162 Action = TLI.getOperationAction(
Node->getOpcode(),
1163 cast<MaskedScatterSDNode>(
Node)->getValue().getValueType());
1166 Action = TLI.getOperationAction(
Node->getOpcode(),
1167 cast<MaskedStoreSDNode>(
Node)->getValue().getValueType());
1169 case ISD::VP_SCATTER:
1170 Action = TLI.getOperationAction(
1172 cast<VPScatterSDNode>(
Node)->getValue().getValueType());
1175 Action = TLI.getOperationAction(
1177 cast<VPStoreSDNode>(
Node)->getValue().getValueType());
1179 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1180 Action = TLI.getOperationAction(
1182 cast<VPStridedStoreSDNode>(
Node)->getValue().getValueType());
1200 Action = TLI.getOperationAction(
1201 Node->getOpcode(),
Node->getOperand(0).getValueType());
1205 case ISD::VP_REDUCE_FADD:
1206 case ISD::VP_REDUCE_FMUL:
1207 case ISD::VP_REDUCE_ADD:
1208 case ISD::VP_REDUCE_MUL:
1209 case ISD::VP_REDUCE_AND:
1210 case ISD::VP_REDUCE_OR:
1211 case ISD::VP_REDUCE_XOR:
1212 case ISD::VP_REDUCE_SMAX:
1213 case ISD::VP_REDUCE_SMIN:
1214 case ISD::VP_REDUCE_UMAX:
1215 case ISD::VP_REDUCE_UMIN:
1216 case ISD::VP_REDUCE_FMAX:
1217 case ISD::VP_REDUCE_FMIN:
1218 case ISD::VP_REDUCE_SEQ_FADD:
1219 case ISD::VP_REDUCE_SEQ_FMUL:
1220 Action = TLI.getOperationAction(
1221 Node->getOpcode(),
Node->getOperand(1).getValueType());
1225 Action = TLI.getCustomOperationAction(*
Node);
1227 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1232 if (SimpleFinishLegalizing) {
1234 switch (
Node->getOpcode()) {
1253 NewNode = DAG.UpdateNodeOperands(
Node, Op0, SAO);
1273 NewNode = DAG.UpdateNodeOperands(
Node, Op0, Op1, SAO);
1279 if (NewNode !=
Node) {
1280 ReplaceNode(
Node, NewNode);
1284 case TargetLowering::Legal:
1287 case TargetLowering::Custom:
1295 if (
Node->getNumValues() == 1) {
1299 Node->getValueType(0) == MVT::Glue) &&
1300 "Type mismatch for custom legalized operation");
1308 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i) {
1312 Node->getValueType(i) == MVT::Glue) &&
1313 "Type mismatch for custom legalized operation");
1317 ReplaceNode(
Node, ResultVals.
data());
1322 case TargetLowering::Expand:
1323 if (ExpandNode(
Node))
1326 case TargetLowering::LibCall:
1327 ConvertNodeToLibcall(
Node);
1329 case TargetLowering::Promote:
1335 switch (
Node->getOpcode()) {
1348 return LegalizeLoadOps(
Node);
1350 return LegalizeStoreOps(
Node);
1354SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(
SDValue Op) {
1374 if (
ST->isIndexed() ||
ST->isTruncatingStore() ||
1375 ST->getValue() != Vec)
1380 if (!
ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1389 ST->hasPredecessor(
Op.getNode()))
1402 StackPtr = DAG.CreateStackTemporary(VecVT);
1405 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO);
1409 Align ElementAlignment =
1410 std::min(cast<StoreSDNode>(Ch)->
getAlign(),
1411 DAG.getDataLayout().getPrefTypeAlign(
1412 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1414 if (
Op.getValueType().isVector()) {
1415 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1416 Op.getValueType(),
Idx);
1417 NewLoad = DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr,
1420 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1421 NewLoad = DAG.getExtLoad(
ISD::EXTLOAD, dl,
Op.getValueType(), Ch, StackPtr,
1427 DAG.ReplaceAllUsesOfValueWith(Ch,
SDValue(NewLoad.
getNode(), 1));
1433 NewLoadOperands[0] = Ch;
1435 SDValue(DAG.UpdateNodeOperands(NewLoad.
getNode(), NewLoadOperands), 0);
1439SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(
SDValue Op) {
1440 assert(
Op.getValueType().isVector() &&
"Non-vector insert subvector!");
1451 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1456 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1459 Idx = DAG.getFreeze(
Idx);
1464 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, PartVT,
Idx);
1468 Ch, dl, Part, SubStackPtr,
1472 TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1475 Ch = DAG.getTruncStore(
1476 Ch, dl, Part, SubStackPtr,
1482 return DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1488 "Unexpected opcode!");
1494 EVT VT =
Node->getValueType(0);
1496 :
Node->getOperand(0).getValueType();
1498 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1499 int FI = cast<FrameIndexSDNode>(FIPtr.
getNode())->getIndex();
1506 assert(TypeByteSize > 0 &&
"Vector element type too small for stack store!");
1510 bool Truncate = isa<BuildVectorSDNode>(
Node) &&
1511 MemVT.
bitsLT(
Node->getOperand(0).getValueType());
1514 for (
unsigned i = 0, e =
Node->getNumOperands(); i != e; ++i) {
1516 if (
Node->getOperand(i).isUndef())
continue;
1518 unsigned Offset = TypeByteSize*i;
1524 Stores.
push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1528 Stores.
push_back(DAG.getStore(DAG.getEntryNode(), dl,
Node->getOperand(i),
1533 if (!Stores.
empty())
1536 StoreChain = DAG.getEntryNode();
1539 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1545void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1548 EVT FloatVT =
Value.getValueType();
1550 State.FloatVT = FloatVT;
1553 if (TLI.isTypeLegal(IVT)) {
1556 State.SignBit = NumBits - 1;
1562 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1565 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1570 State.Chain = DAG.getStore(DAG.getEntryNode(),
DL,
Value, State.FloatPtr,
1571 State.FloatPointerInfo);
1578 State.IntPointerInfo = State.FloatPointerInfo;
1581 unsigned ByteOffset = (NumBits / 8) - 1;
1588 State.IntPtr = IntPtr;
1589 State.IntValue = DAG.getExtLoad(
ISD::EXTLOAD,
DL, LoadTy, State.Chain, IntPtr,
1590 State.IntPointerInfo, MVT::i8);
1597SDValue SelectionDAGLegalize::modifySignAsInt(
const FloatSignAsInt &State,
1604 SDValue Chain = DAG.getTruncStore(State.Chain,
DL, NewIntValue, State.IntPtr,
1605 State.IntPointerInfo, MVT::i8);
1606 return DAG.getLoad(State.FloatVT,
DL, Chain, State.FloatPtr,
1607 State.FloatPointerInfo);
1616 FloatSignAsInt SignAsInt;
1617 getSignAsIntValue(SignAsInt,
DL, Sign);
1619 EVT IntVT = SignAsInt.IntValue.getValueType();
1620 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1626 if (TLI.isOperationLegalOrCustom(
ISD::FABS, FloatVT) &&
1627 TLI.isOperationLegalOrCustom(
ISD::FNEG, FloatVT)) {
1630 SDValue Cond = DAG.getSetCC(
DL, getSetCCResultType(IntVT), SignBit,
1632 return DAG.getSelect(
DL, FloatVT,
Cond, NegValue, AbsValue);
1636 FloatSignAsInt MagAsInt;
1637 getSignAsIntValue(MagAsInt,
DL, Mag);
1638 EVT MagVT = MagAsInt.IntValue.getValueType();
1639 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask,
DL, MagVT);
1644 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1645 EVT ShiftVT = IntVT;
1651 if (ShiftAmount > 0) {
1652 SDValue ShiftCnst = DAG.getConstant(ShiftAmount,
DL, ShiftVT);
1654 }
else if (ShiftAmount < 0) {
1655 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount,
DL, ShiftVT);
1665 return modifySignAsInt(MagAsInt,
DL, CopiedSign);
1671 FloatSignAsInt SignAsInt;
1672 getSignAsIntValue(SignAsInt,
DL,
Node->getOperand(0));
1673 EVT IntVT = SignAsInt.IntValue.getValueType();
1676 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1681 return modifySignAsInt(SignAsInt,
DL, SignFlip);
1689 EVT FloatVT =
Value.getValueType();
1696 FloatSignAsInt ValueAsInt;
1697 getSignAsIntValue(ValueAsInt,
DL,
Value);
1698 EVT IntVT = ValueAsInt.IntValue.getValueType();
1699 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask,
DL, IntVT);
1702 return modifySignAsInt(ValueAsInt,
DL, ClearedSign);
1705void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(
SDNode*
Node,
1707 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1708 assert(SPReg &&
"Target cannot require DYNAMIC_STACKALLOC expansion and"
1709 " not tell us which reg is the stack pointer!");
1711 EVT VT =
Node->getValueType(0);
1719 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1722 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1724 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1732 if (Alignment > StackAlign)
1734 DAG.getConstant(-Alignment.
value(), dl, VT));
1735 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);
1737 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0,
SDValue(), dl);
1749 return EmitStackConvert(
SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1757 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1760 if ((SrcVT.
bitsGT(SlotVT) &&
1761 !TLI.isTruncStoreLegalOrCustom(
SrcOp.getValueType(), SlotVT)) ||
1762 (SlotVT.
bitsLT(DestVT) &&
1763 !TLI.isLoadExtLegalOrCustom(
ISD::EXTLOAD, DestVT, SlotVT)))
1767 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1768 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1780 if (SrcVT.
bitsGT(SlotVT))
1781 Store = DAG.getTruncStore(Chain, dl,
SrcOp, FIPtr, PtrInfo,
1785 Store = DAG.getStore(Chain, dl,
SrcOp, FIPtr, PtrInfo, SrcAlign);
1789 if (SlotVT.
bitsEq(DestVT))
1790 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1793 return DAG.getExtLoad(
ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1806 SDValue Ch = DAG.getTruncStore(
1807 DAG.getEntryNode(), dl,
Node->getOperand(0), StackPtr,
1809 Node->getValueType(0).getVectorElementType());
1811 Node->getValueType(0), dl, Ch, StackPtr,
1818 unsigned NumElems =
Node->getNumOperands();
1820 EVT VT =
Node->getValueType(0);
1832 for (
unsigned i = 0; i < NumElems; ++i) {
1843 while (IntermedVals.
size() > 2) {
1844 NewIntermedVals.
clear();
1845 for (
unsigned i = 0, e = (IntermedVals.
size() & ~1u); i < e; i += 2) {
1851 FinalIndices.
reserve(IntermedVals[i].second.
size() +
1852 IntermedVals[i+1].second.
size());
1855 for (
unsigned j = 0, f = IntermedVals[i].second.
size(); j != f;
1858 FinalIndices.
push_back(IntermedVals[i].second[j]);
1860 for (
unsigned j = 0, f = IntermedVals[i+1].second.
size(); j != f;
1862 ShuffleVec[k] = NumElems + j;
1863 FinalIndices.
push_back(IntermedVals[i+1].second[j]);
1869 IntermedVals[i+1].first,
1874 std::make_pair(Shuffle, std::move(FinalIndices)));
1879 if ((IntermedVals.
size() & 1) != 0)
1882 IntermedVals.
swap(NewIntermedVals);
1886 "Invalid number of intermediate vectors");
1887 SDValue Vec1 = IntermedVals[0].first;
1889 if (IntermedVals.
size() > 1)
1890 Vec2 = IntermedVals[1].first;
1895 for (
unsigned i = 0, e = IntermedVals[0].second.
size(); i != e; ++i)
1896 ShuffleVec[IntermedVals[0].second[i]] = i;
1897 for (
unsigned i = 0, e = IntermedVals[1].second.
size(); i != e; ++i)
1898 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1912 unsigned NumElems =
Node->getNumOperands();
1915 EVT VT =
Node->getValueType(0);
1916 EVT OpVT =
Node->getOperand(0).getValueType();
1921 bool isOnlyLowElement =
true;
1922 bool MoreThanTwoValues =
false;
1924 for (
unsigned i = 0; i < NumElems; ++i) {
1929 isOnlyLowElement =
false;
1930 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1935 }
else if (!Value2.
getNode()) {
1938 }
else if (V != Value1 && V != Value2) {
1939 MoreThanTwoValues =
true;
1944 return DAG.getUNDEF(VT);
1946 if (isOnlyLowElement)
1952 for (
unsigned i = 0, e = NumElems; i !=
e; ++i) {
1954 dyn_cast<ConstantFPSDNode>(
Node->getOperand(i))) {
1957 dyn_cast<ConstantSDNode>(
Node->getOperand(i))) {
1976 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1977 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1979 VT, dl, DAG.getEntryNode(), CPIdx,
1985 for (
unsigned i = 0; i < NumElems; ++i) {
1986 if (
Node->getOperand(i).isUndef())
1991 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.
size())) {
1992 if (!MoreThanTwoValues) {
1994 for (
unsigned i = 0; i < NumElems; ++i) {
1998 ShuffleVec[i] =
V == Value1 ? 0 : NumElems;
2000 if (TLI.isShuffleMaskLegal(ShuffleVec,
Node->getValueType(0))) {
2007 Vec2 = DAG.getUNDEF(VT);
2010 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2020 return ExpandVectorBuildThroughStack(
Node);
2025 EVT VT =
Node->getValueType(0);
2028 return DAG.getSplatBuildVector(VT,
DL, SplatVal);
2039 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2040 TLI.getPointerTy(DAG.getDataLayout()));
2042 EVT RetVT =
Node->getValueType(0);
2049 SDValue InChain = DAG.getEntryNode();
2054 const Function &
F = DAG.getMachineFunction().getFunction();
2056 TLI.isInTailCallPosition(DAG,
Node, TCChain) &&
2057 (
RetTy ==
F.getReturnType() ||
F.getReturnType()->isVoidTy());
2062 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT,
isSigned);
2065 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2067 .setTailCall(isTailCall)
2068 .setSExtResult(signExtend)
2069 .setZExtResult(!signExtend)
2070 .setIsPostTypeLegalization(
true);
2072 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2077 return {DAG.getRoot(), DAG.getRoot()};
2089 EVT ArgVT =
Op.getValueType();
2093 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT,
isSigned);
2094 Entry.IsZExt = !Entry.IsSExt;
2095 Args.push_back(Entry);
2098 return ExpandLibCall(LC,
Node, std::move(Args),
isSigned);
2101void SelectionDAGLegalize::ExpandFrexpLibCall(
2104 EVT VT =
Node->getValueType(0);
2105 EVT ExpVT =
Node->getValueType(1);
2113 FPArgEntry.
Node = FPOp;
2114 FPArgEntry.
Ty = ArgTy;
2116 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2118 PtrArgEntry.
Node = StackSlot;
2119 PtrArgEntry.
Ty = PointerType::get(*DAG.getContext(),
2120 DAG.getDataLayout().getAllocaAddrSpace());
2125 auto [
Call, Chain] = ExpandLibCall(LC,
Node, std::move(Args),
false);
2129 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2133 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2135 LoadExp.
getValue(1), DAG.getRoot());
2136 DAG.setRoot(OutputChain);
2142void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2145 if (LC == RTLIB::UNKNOWN_LIBCALL)
2148 if (
Node->isStrictFPOpcode()) {
2149 EVT RetVT =
Node->getValueType(0);
2153 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2156 Node->getOperand(0));
2158 Results.push_back(Tmp.second);
2160 SDValue Tmp = ExpandLibCall(LC,
Node,
false).first;
2166void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2174 Call_F32, Call_F64, Call_F80,
2175 Call_F128, Call_PPCF128);
2186 switch (
Node->getSimpleValueType(0).SimpleTy) {
2188 case MVT::i8: LC = Call_I8;
break;
2189 case MVT::i16: LC = Call_I16;
break;
2190 case MVT::i32: LC = Call_I32;
break;
2191 case MVT::i64: LC = Call_I64;
break;
2192 case MVT::i128: LC = Call_I128;
break;
2199void SelectionDAGLegalize::ExpandArgFPLibCall(
SDNode*
Node,
2206 EVT InVT =
Node->getOperand(
Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2208 Call_F32, Call_F64, Call_F80,
2209 Call_F128, Call_PPCF128);
2215SelectionDAGLegalize::ExpandDivRemLibCall(
SDNode *
Node,
2217 unsigned Opcode =
Node->getOpcode();
2221 switch (
Node->getSimpleValueType(0).SimpleTy) {
2223 case MVT::i8: LC=
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
2224 case MVT::i16: LC=
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
2225 case MVT::i32: LC=
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
2226 case MVT::i64: LC=
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
2227 case MVT::i128: LC=
isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
2233 SDValue InChain = DAG.getEntryNode();
2235 EVT RetVT =
Node->getValueType(0);
2241 EVT ArgVT =
Op.getValueType();
2247 Args.push_back(Entry);
2251 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2253 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2256 Args.push_back(Entry);
2258 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2259 TLI.getPointerTy(DAG.getDataLayout()));
2265 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2270 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2282 switch (
Node->getSimpleValueType(0).SimpleTy) {
2284 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2285 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2286 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2287 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2288 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2311SelectionDAGLegalize::ExpandSinCosLibCall(
SDNode *
Node,
2314 switch (
Node->getSimpleValueType(0).SimpleTy) {
2316 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2317 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2318 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2319 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2320 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2326 SDValue InChain = DAG.getEntryNode();
2328 EVT RetVT =
Node->getValueType(0);
2335 Entry.Node =
Node->getOperand(0);
2337 Entry.IsSExt =
false;
2338 Entry.IsZExt =
false;
2339 Args.push_back(Entry);
2342 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2343 Entry.Node = SinPtr;
2344 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2345 Entry.IsSExt =
false;
2346 Entry.IsZExt =
false;
2347 Args.push_back(Entry);
2350 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2351 Entry.Node = CosPtr;
2352 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2353 Entry.IsSExt =
false;
2354 Entry.IsZExt =
false;
2355 Args.push_back(Entry);
2357 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2358 TLI.getPointerTy(DAG.getDataLayout()));
2362 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2363 TLI.getLibcallCallingConv(LC),
Type::getVoidTy(*DAG.getContext()), Callee,
2366 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2376 EVT VT =
Node->getValueType(0);
2379 EVT ExpVT =
N.getValueType();
2381 if (AsIntVT ==
EVT())
2394 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2399 const int Precision = APFloat::semanticsPrecision(FltSem);
2401 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2402 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2404 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2406 const APFloat One(FltSem,
"1.0");
2407 APFloat ScaleUpK =
scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2411 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2420 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2426 DAG.getSetCC(dl, SetCCVT,
N, DoubleMaxExp,
ISD::SETUGT);
2428 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2440 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2441 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2446 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2451 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2455 SDValue ScaleDownTwice = DAG.getSetCC(
2456 dl, SetCCVT,
N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2468 DAG.getNode(
ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small,
X));
2472 DAG.getNode(
ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small,
N));
2477 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2478 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2481 ExponentShiftAmt, NUW_NSW);
2483 return DAG.getNode(
ISD::FMUL, dl, VT, NewX, AsFP);
2490 EVT ExpVT =
Node->getValueType(1);
2492 if (AsIntVT ==
EVT())
2497 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2514 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2518 SDValue SmallestNormalizedInt = DAG.getConstant(
2524 DAG.getConstant(
APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2530 FractSignMaskVal.
setBit(BitSize - 1);
2533 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2535 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2537 const APFloat One(FltSem,
"1.0");
2541 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2543 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2547 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2553 SDValue AddNegSmallestNormal =
2555 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2559 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt,
ISD::SETULT);
2561 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2576 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2579 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2582 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2589 SDValue FPHalf = DAG.getConstant(
Half.bitcastToAPInt(), dl, AsIntVT);
2602 return DAG.getMergeValues({Result0, Result1}, dl);
2613 EVT DestVT =
Node->getValueType(0);
2615 unsigned OpNo =
Node->isStrictFPOpcode() ? 1 : 0;
2621 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2622 (DestVT.
bitsLE(MVT::f64) ||
2626 LLVM_DEBUG(
dbgs() <<
"32-bit [signed|unsigned] integer to float/double "
2630 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2637 DAG.getConstant(0x80000000u, dl, MVT::i32));
2640 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2643 if (DAG.getDataLayout().isBigEndian())
2646 SDValue MemChain = DAG.getEntryNode();
2649 SDValue Store1 = DAG.getStore(MemChain, dl,
Lo, StackSlot,
2662 SDValue Bias = DAG.getConstantFP(
2663 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2664 : llvm::bit_cast<double>(0x4330000000000000ULL),
2669 if (
Node->isStrictFPOpcode()) {
2671 {
Node->getOperand(0), Load, Bias});
2674 std::pair<SDValue, SDValue> ResultPair;
2676 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2677 Result = ResultPair.first;
2678 Chain = ResultPair.second;
2684 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2693 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2694 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2695 LLVM_DEBUG(
dbgs() <<
"Converting unsigned i32/i64 to f32/f64\n");
2710 EVT SetCCVT = getSetCCResultType(SrcVT);
2712 SDValue SignBitTest = DAG.getSetCC(
2713 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT),
ISD::SETLT);
2715 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2716 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2718 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2723 if (
Node->isStrictFPOpcode()) {
2726 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest,
Or, Op0);
2728 {
Node->getOperand(0), InCvt });
2736 Flags.setNoFPExcept(
Node->getFlags().hasNoFPExcept());
2737 Fast->setFlags(Flags);
2738 Flags.setNoFPExcept(
true);
2746 return DAG.getSelect(dl, DestVT, SignBitTest, Slow,
Fast);
2750 if (!TLI.isOperationLegalOrCustom(
2757 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2759 "Cannot perform lossless SINT_TO_FP!");
2762 if (
Node->isStrictFPOpcode()) {
2764 {
Node->getOperand(0), Op0 });
2768 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2771 Four = DAG.getIntPtrConstant(4, dl);
2772 SDValue CstOffset = DAG.getSelect(dl,
Zero.getValueType(),
2773 SignSet, Four, Zero);
2782 case MVT::i8 : FF = 0x43800000ULL;
break;
2783 case MVT::i16: FF = 0x47800000ULL;
break;
2784 case MVT::i32: FF = 0x4F800000ULL;
break;
2785 case MVT::i64: FF = 0x5F800000ULL;
break;
2787 if (DAG.getDataLayout().isLittleEndian())
2789 Constant *FudgeFactor = ConstantInt::get(
2793 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2794 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2798 if (DestVT == MVT::f32)
2799 FudgeInReg = DAG.getLoad(
2800 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2809 LegalizeOp(
Load.getNode());
2813 if (
Node->isStrictFPOpcode()) {
2815 { Tmp1.
getValue(1), Tmp1, FudgeInReg });
2816 Chain =
Result.getValue(1);
2820 return DAG.getNode(
ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2828void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2830 bool IsStrict =
N->isStrictFPOpcode();
2833 EVT DestVT =
N->getValueType(0);
2834 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2841 unsigned OpToUse = 0;
2849 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2857 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2869 DAG.
getNode(OpToUse, dl, {DestVT, MVT::Other},
2872 dl, NewInTy, LegalOp)});
2879 DAG.getNode(OpToUse, dl, DestVT,
2881 dl, NewInTy, LegalOp)));
2889void SelectionDAGLegalize::PromoteLegalFP_TO_INT(
SDNode *
N,
const SDLoc &dl,
2891 bool IsStrict =
N->isStrictFPOpcode();
2894 EVT DestVT =
N->getValueType(0);
2895 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2897 EVT NewOutTy = DestVT;
2899 unsigned OpToUse = 0;
2909 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2914 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2923 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2924 Operation = DAG.getNode(OpToUse, dl, VTs,
N->getOperand(0), LegalOp);
2926 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2941 unsigned Opcode =
Node->getOpcode();
2944 EVT NewOutTy =
Node->getValueType(0);
2949 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2956 Node->getOperand(1));
2962 EVT VT =
Op.getValueType();
2963 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2968 if (TLI.isOperationLegalOrPromote(
ISD::CTPOP, VT)) {
2974 DAG.getConstant(1ULL << (--i), dl, ShVT));
2979 return DAG.getNode(
ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2982bool SelectionDAGLegalize::ExpandNode(
SDNode *
Node) {
2986 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2988 switch (
Node->getOpcode()) {
2990 if ((Tmp1 = TLI.expandABS(
Node, DAG)))
2995 if ((Tmp1 = TLI.expandABD(
Node, DAG)))
2999 if ((Tmp1 = TLI.expandCTPOP(
Node, DAG)))
3004 if ((Tmp1 = TLI.expandCTLZ(
Node, DAG)))
3009 if ((Tmp1 = TLI.expandCTTZ(
Node, DAG)))
3013 if ((Tmp1 = TLI.expandBITREVERSE(
Node, DAG)))
3017 if ((Tmp1 = TLI.expandBSWAP(
Node, DAG)))
3021 Results.push_back(ExpandPARITY(
Node->getOperand(0), dl));
3026 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3029 SDValue CfaArg = DAG.getSExtOrTrunc(
Node->getOperand(0), dl,
3030 TLI.getPointerTy(DAG.getDataLayout()));
3038 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3044 Results.push_back(DAG.getConstant(1, dl,
Node->getValueType(0)));
3061 DAG.getConstant(0, dl,
Node->getValueType(0)));
3067 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3073 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3074 SDValue Swap = DAG.getAtomicCmpSwap(
3076 Node->getOperand(0),
Node->getOperand(1), Zero, Zero,
3077 cast<AtomicSDNode>(
Node)->getMemOperand());
3086 Node->getOperand(0),
Node->getOperand(2),
Node->getOperand(1),
3087 cast<AtomicSDNode>(
Node)->getMemOperand());
3095 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3096 SDValue Res = DAG.getAtomicCmpSwap(
3098 Node->getOperand(0),
Node->getOperand(1),
Node->getOperand(2),
3099 Node->getOperand(3), cast<MemSDNode>(
Node)->getMemOperand());
3105 EVT AtomicType = cast<AtomicSDNode>(
Node)->getMemoryVT();
3106 EVT OuterType =
Node->getValueType(0);
3107 switch (TLI.getExtendForAtomicOps()) {
3110 DAG.getValueType(AtomicType));
3112 Node->getOperand(2), DAG.getValueType(AtomicType));
3117 DAG.getValueType(AtomicType));
3118 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3122 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3123 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3139 EVT VT =
Node->getValueType(0);
3143 cast<VTSDNode>(
RHS->getOperand(1))->getVT() == AN->
getMemoryVT())
3144 RHS =
RHS->getOperand(0);
3148 Node->getOperand(0),
Node->getOperand(1),
3158 for (
unsigned i = 0; i <
Node->getNumValues(); i++)
3162 EVT VT =
Node->getValueType(0);
3164 Results.push_back(DAG.getConstant(0, dl, VT));
3167 Results.push_back(DAG.getConstantFP(0, dl, VT));
3174 if (TLI.isStrictFPEnabled())
3178 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3179 Node->getValueType(0))
3180 == TargetLowering::Legal)
3184 if ((Tmp1 = EmitStackConvert(
Node->getOperand(1),
Node->getValueType(0),
3185 Node->getValueType(0), dl,
3186 Node->getOperand(0)))) {
3188 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_ROUND node\n");
3193 if ((Tmp1 = TLI.expandFP_ROUND(
Node, DAG))) {
3201 if ((Tmp1 = EmitStackConvert(
Node->getOperand(0),
Node->getValueType(0),
3202 Node->getValueType(0), dl)))
3208 if (TLI.isStrictFPEnabled())
3212 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3213 Node->getValueType(0))
3214 == TargetLowering::Legal)
3218 if ((Tmp1 = EmitStackConvert(
3219 Node->getOperand(1),
Node->getOperand(1).getValueType(),
3220 Node->getValueType(0), dl,
Node->getOperand(0)))) {
3222 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_EXTEND node\n");
3228 EVT SrcVT =
Op.getValueType();
3229 EVT DstVT =
Node->getValueType(0);
3235 if ((Tmp1 = EmitStackConvert(
Op, SrcVT, DstVT, dl)))
3245 if (
Op.getValueType() == MVT::bf16) {
3249 Op = DAG.getAnyExtOrTrunc(
Op, dl, MVT::i32);
3253 DAG.getConstant(16, dl,
3254 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3257 if (
Node->getValueType(0) != MVT::f32)
3264 if (
Op.getValueType() != MVT::f32)
3266 DAG.getIntPtrConstant(0, dl,
true));
3268 if (!DAG.isKnownNeverSNaN(
Op)) {
3273 DAG.getConstant(16, dl,
3274 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3277 if (
Node->getValueType(0) == MVT::bf16) {
3281 Op = DAG.getAnyExtOrTrunc(
Op, dl,
Node->getValueType(0));
3287 EVT ExtraVT = cast<VTSDNode>(
Node->getOperand(1))->getVT();
3288 EVT VT =
Node->getValueType(0);
3298 SDValue One = DAG.getConstant(1, dl, VT);
3308 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3311 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3313 Node->getOperand(0), ShiftCst);
3320 if (TLI.expandUINT_TO_FP(
Node, Tmp1, Tmp2, DAG)) {
3322 if (
Node->isStrictFPOpcode())
3329 if ((Tmp1 = ExpandLegalINT_TO_FP(
Node, Tmp2))) {
3331 if (
Node->isStrictFPOpcode())
3336 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG))
3340 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG)) {
3342 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_SINT node\n");
3347 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG))
3351 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG)) {
3353 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node,1), Tmp2);
3356 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_UINT node\n");
3362 Results.push_back(TLI.expandFP_TO_INT_SAT(
Node, DAG));
3372 if (
Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3375 Node->getOperand(0));
3377 Tmp1 = ExpandExtractFromVectorThroughStack(
SDValue(
Node, 0));
3387 Results.push_back(ExpandVectorBuildThroughStack(
Node));
3399 EVT VT =
Node->getValueType(0);
3403 if (!TLI.isTypeLegal(EltVT)) {
3404 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3409 if (NewEltVT.
bitsLT(EltVT)) {
3425 unsigned int factor =
3433 for (
unsigned fi = 0; fi < factor; ++fi)
3437 for (
unsigned fi = 0; fi < factor; ++fi)
3448 for (
unsigned i = 0; i != NumElems; ++i) {
3456 DAG.getVectorIdxConstant(
Idx, dl)));
3460 DAG.getVectorIdxConstant(
Idx - NumElems, dl)));
3463 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3470 Results.push_back(TLI.expandVectorSplice(
Node, DAG));
3474 EVT OpTy =
Node->getOperand(0).getValueType();
3475 if (
Node->getConstantOperandVal(1)) {
3479 TLI.getShiftAmountTy(
3480 Node->getOperand(0).getValueType(),
3481 DAG.getDataLayout())));
3486 Node->getOperand(0));
3494 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3495 Results.push_back(DAG.getCopyFromReg(
Node->getOperand(0), dl, SP,
3496 Node->getValueType(0)));
3499 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
3506 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3507 Results.push_back(DAG.getCopyToReg(
Node->getOperand(0), dl, SP,
3508 Node->getOperand(1)));
3514 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3529 TLI.expandIS_FPCLASS(
Node->getValueType(0),
Node->getOperand(0),
3540 switch (
Node->getOpcode()) {
3547 Tmp1 =
Node->getOperand(0);
3548 Tmp2 =
Node->getOperand(1);
3549 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3555 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG))
3561 EVT VT =
Node->getValueType(0);
3567 SDVTList VTs = DAG.getVTList(VT, VT);
3577 EVT VT =
Node->getValueType(0);
3581 if (TLI.getLibcallName(LC))
3587 Results.push_back(Expanded.getValue(1));
3596 if (TLI.getLibcallName(LC))
3601 Results.push_back(Expanded.getValue(1));
3609 if (
Node->getValueType(0) != MVT::f32) {
3621 if (
Node->getValueType(0) != MVT::f32) {
3626 {Node->getOperand(0), Node->getOperand(1)});
3628 {
Node->getValueType(0), MVT::Other},
3636 if (!TLI.useSoftFloat() &&
TM.Options.UnsafeFPMath) {
3638 MVT SVT =
Op.getSimpleValueType();
3639 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3645 DAG.getIntPtrConstant(0, dl,
true));
3656 DAG.shouldOptForSize()))
3657 Results.push_back(ExpandConstantFP(CFP,
true));
3662 Results.push_back(ExpandConstant(CP));
3666 EVT VT =
Node->getValueType(0);
3667 if (TLI.isOperationLegalOrCustom(
ISD::FADD, VT) &&
3668 TLI.isOperationLegalOrCustom(
ISD::FNEG, VT)) {
3677 EVT VT =
Node->getValueType(0);
3679 TLI.isOperationLegalOrCustom(
ISD::XOR, VT) &&
3680 "Don't know how to expand this subtraction!");
3681 Tmp1 = DAG.getNOT(dl,
Node->getOperand(1), VT);
3682 Tmp1 = DAG.
getNode(
ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3688 if (TLI.expandREM(
Node, Tmp1, DAG))
3695 EVT VT =
Node->getValueType(0);
3696 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3697 SDVTList VTs = DAG.getVTList(VT, VT);
3698 Tmp1 = DAG.
getNode(DivRemOpc, dl, VTs,
Node->getOperand(0),
3699 Node->getOperand(1));
3706 unsigned ExpandOpcode =
3708 EVT VT =
Node->getValueType(0);
3709 SDVTList VTs = DAG.getVTList(VT, VT);
3711 Tmp1 = DAG.
getNode(ExpandOpcode, dl, VTs,
Node->getOperand(0),
3712 Node->getOperand(1));
3720 MVT VT =
LHS.getSimpleValueType();
3721 unsigned MULHOpcode =
3724 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3726 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3732 assert(TLI.isTypeLegal(HalfType));
3733 if (TLI.expandMUL_LOHI(
Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3735 TargetLowering::MulExpansionKind::Always)) {
3736 for (
unsigned i = 0; i < 2; ++i) {
3739 SDValue Shift = DAG.getConstant(
3741 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3750 EVT VT =
Node->getValueType(0);
3751 SDVTList VTs = DAG.getVTList(VT, VT);
3757 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::SMUL_LOHI, VT);
3758 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::UMUL_LOHI, VT);
3759 bool HasMULHS = TLI.isOperationLegalOrCustom(
ISD::MULHS, VT);
3760 bool HasMULHU = TLI.isOperationLegalOrCustom(
ISD::MULHU, VT);
3761 unsigned OpToUse = 0;
3762 if (HasSMUL_LOHI && !HasMULHS) {
3764 }
else if (HasUMUL_LOHI && !HasMULHU) {
3766 }
else if (HasSMUL_LOHI) {
3768 }
else if (HasUMUL_LOHI) {
3772 Results.push_back(DAG.getNode(OpToUse, dl, VTs,
Node->getOperand(0),
3773 Node->getOperand(1)));
3781 TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
3782 TLI.isOperationLegalOrCustom(
ISD::OR, VT) &&
3783 TLI.expandMUL(
Node,
Lo,
Hi, HalfType, DAG,
3784 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3789 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3797 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG))
3802 if (
SDValue Expanded = TLI.expandROT(
Node,
true , DAG))
3809 Results.push_back(TLI.expandAddSubSat(
Node, DAG));
3819 Results.push_back(TLI.expandFixedPointMul(
Node, DAG));
3826 Node->getOperand(0),
3827 Node->getOperand(1),
3828 Node->getConstantOperandVal(2),
3851 EVT VT =
LHS.getValueType();
3855 EVT CarryType =
Node->getValueType(1);
3856 EVT SetCCType = getSetCCResultType(
Node->getValueType(0));
3858 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS,
CC);
3861 SDValue One = DAG.getConstant(1, dl, VT);
3863 DAG.
getNode(
ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3872 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero,
ISD::SETEQ)
3873 : DAG.getSetCC(dl, SetCCType, Sum, Zero,
ISD::SETEQ);
3875 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3881 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3887 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
3895 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
3903 if (TLI.expandMULO(
Node, Result, Overflow, DAG)) {
3915 DAG.getConstant(
PairTy.getSizeInBits() / 2, dl,
3916 TLI.getShiftAmountTy(
PairTy, DAG.getDataLayout())));
3921 Tmp1 =
Node->getOperand(0);
3922 Tmp2 =
Node->getOperand(1);
3923 Tmp3 =
Node->getOperand(2);
3927 cast<CondCodeSDNode>(Tmp1.
getOperand(2))->get());
3929 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3940 int JTI = cast<JumpTableSDNode>(Table.
getNode())->getIndex();
3943 EVT PTy = TLI.getPointerTy(TD);
3945 unsigned EntrySize =
3946 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3953 Index = DAG.getNode(
3958 DAG.getConstant(EntrySize, dl,
Index.getValueType()));
3967 if (TLI.isJumpTableRelative()) {
3972 TLI.getPICJumpTableRelocBase(Table, DAG));
3975 Tmp1 = TLI.expandIndirectJTBranch(dl,
LD.getValue(1),
Addr, JTI, DAG);
3982 Tmp1 =
Node->getOperand(0);
3983 Tmp2 =
Node->getOperand(1);
3989 Node->getOperand(2));
4001 Node->getOperand(2));
4009 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
4014 unsigned Offset = IsStrict ? 1 : 0;
4023 bool Legalized = TLI.LegalizeSetCCCondCode(
4024 DAG,
Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4025 Chain, IsSignaling);
4033 {Chain, Tmp1, Tmp2, Tmp3},
Node->getFlags());
4037 {Tmp1, Tmp2, Tmp3, Mask, EVL},
Node->getFlags());
4039 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl,
Node->getValueType(0), Tmp1,
4040 Tmp2, Tmp3,
Node->getFlags());
4048 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->
getValueType(0));
4051 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->
getValueType(0));
4063 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
4068 EVT VT =
Node->getValueType(0);
4071 DAG.getBoolConstant(
true, dl, VT, Tmp1VT),
4072 DAG.getBoolConstant(
false, dl, VT, Tmp1VT), Tmp3);
4079 Tmp1 =
Node->getOperand(0);
4080 Tmp2 =
Node->getOperand(1);
4081 Tmp3 =
Node->getOperand(2);
4082 Tmp4 =
Node->getOperand(3);
4083 EVT VT =
Node->getValueType(0);
4093 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4095 EVT CCVT = getSetCCResultType(CmpVT);
4097 Results.push_back(DAG.getSelect(dl, VT,
Cond, Tmp3, Tmp4));
4102 bool Legalized =
false;
4110 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4120 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4126 Legalized = TLI.LegalizeSetCCCondCode(
4130 assert(Legalized &&
"Can't legalize SELECT_CC with legal condition!");
4141 Tmp1, Tmp2, Tmp3, Tmp4,
CC);
4146 Tmp2, Tmp3, Tmp4,
CC);
4156 Tmp1 =
Node->getOperand(0);
4157 Tmp2 =
Node->getOperand(2);
4158 Tmp3 =
Node->getOperand(3);
4159 Tmp4 =
Node->getOperand(1);
4161 bool Legalized = TLI.LegalizeSetCCCondCode(
4162 DAG, getSetCCResultType(Tmp2.
getValueType()), Tmp2, Tmp3, Tmp4,
4165 assert(Legalized &&
"Can't legalize BR_CC with legal condition!");
4170 assert(!NeedInvert &&
"Don't know how to invert BR_CC!");
4173 Tmp4, Tmp2, Tmp3,
Node->getOperand(4));
4178 Tmp2, Tmp3,
Node->getOperand(4));
4193 EVT VT =
Node->getValueType(0);
4199 for (
unsigned Idx = 0;
Idx < NumElem;
Idx++) {
4202 Node->getOperand(0), DAG.getVectorIdxConstant(
Idx, dl));
4205 Node->getOperand(1), DAG.getVectorIdxConstant(
Idx, dl));
4229 Results.push_back(TLI.expandVecReduce(
Node, DAG));
4246 if (!TLI.isStrictFPEnabled() &&
Results.
empty() &&
Node->isStrictFPOpcode()) {
4252 switch (
Node->getOpcode()) {
4254 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4255 Node->getValueType(0))
4256 == TargetLowering::Legal)
4260 if (TLI.getStrictFPOperationAction(
4263 if (TLI.getStrictFPOperationAction(
4267 EVT VT =
Node->getValueType(0);
4271 {Node->getOperand(0), Node->getOperand(1), Neg},
4286 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4287 Node->getOperand(1).getValueType())
4288 == TargetLowering::Legal)
4305void SelectionDAGLegalize::ConvertNodeToLibcall(
SDNode *
Node) {
4310 unsigned Opc =
Node->getOpcode();
4319 .setChain(
Node->getOperand(0))
4322 DAG.getExternalSymbol(
"__sync_synchronize",
4323 TLI.getPointerTy(DAG.getDataLayout())),
4326 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4328 Results.push_back(CallResult.second);
4350 EVT RetVT =
Node->getValueType(0);
4353 if (TLI.getLibcallName(LC)) {
4360 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4361 "Unexpected atomic op or value type!");
4365 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4368 Node->getOperand(0));
4370 Results.push_back(Tmp.second);
4378 .setChain(
Node->getOperand(0))
4380 DAG.getExternalSymbol(
4381 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4383 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4385 Results.push_back(CallResult.second);
4390 ExpandFPLibCall(
Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4391 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4392 RTLIB::FMIN_PPCF128,
Results);
4399 ExpandFPLibCall(
Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4400 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4401 RTLIB::FMAX_PPCF128,
Results);
4405 ExpandFPLibCall(
Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4406 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4407 RTLIB::SQRT_PPCF128,
Results);
4410 ExpandFPLibCall(
Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4411 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4412 RTLIB::CBRT_PPCF128,
Results);
4416 ExpandFPLibCall(
Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4417 RTLIB::SIN_F80, RTLIB::SIN_F128,
4422 ExpandFPLibCall(
Node, RTLIB::COS_F32, RTLIB::COS_F64,
4423 RTLIB::COS_F80, RTLIB::COS_F128,
4432 ExpandFPLibCall(
Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4433 RTLIB::LOG_F128, RTLIB::LOG_PPCF128,
Results);
4437 ExpandFPLibCall(
Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4438 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128,
Results);
4442 ExpandFPLibCall(
Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4443 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128,
Results);
4447 ExpandFPLibCall(
Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4448 RTLIB::EXP_F128, RTLIB::EXP_PPCF128,
Results);
4452 ExpandFPLibCall(
Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4453 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128,
Results);
4456 ExpandFPLibCall(
Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4457 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128,
Results);
4461 ExpandFPLibCall(
Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4462 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4463 RTLIB::TRUNC_PPCF128,
Results);
4467 ExpandFPLibCall(
Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4468 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4469 RTLIB::FLOOR_PPCF128,
Results);
4473 ExpandFPLibCall(
Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4474 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4475 RTLIB::CEIL_PPCF128,
Results);
4479 ExpandFPLibCall(
Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4480 RTLIB::RINT_F80, RTLIB::RINT_F128,
4481 RTLIB::RINT_PPCF128,
Results);
4485 ExpandFPLibCall(
Node, RTLIB::NEARBYINT_F32,
4486 RTLIB::NEARBYINT_F64,
4487 RTLIB::NEARBYINT_F80,
4488 RTLIB::NEARBYINT_F128,
4489 RTLIB::NEARBYINT_PPCF128,
Results);
4493 ExpandFPLibCall(
Node, RTLIB::ROUND_F32,
4497 RTLIB::ROUND_PPCF128,
Results);
4501 ExpandFPLibCall(
Node, RTLIB::ROUNDEVEN_F32,
4502 RTLIB::ROUNDEVEN_F64,
4503 RTLIB::ROUNDEVEN_F80,
4504 RTLIB::ROUNDEVEN_F128,
4505 RTLIB::ROUNDEVEN_PPCF128,
Results);
4509 ExpandFPLibCall(
Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4510 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128,
Results);
4519 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
4520 if (!TLI.getLibcallName(LC)) {
4522 if (
Node->isStrictFPOpcode()) {
4525 {
Node->getValueType(0),
Node->getValueType(1)},
4526 {
Node->getOperand(0),
Node->getOperand(2)});
4529 {
Node->getValueType(0),
Node->getValueType(1)},
4536 Node->getOperand(1));
4538 Node->getValueType(0),
4543 unsigned Offset =
Node->isStrictFPOpcode() ? 1 : 0;
4544 bool ExponentHasSizeOfInt =
4545 DAG.getLibInfo().getIntSize() ==
4546 Node->getOperand(1 +
Offset).getValueType().getSizeInBits();
4547 if (!ExponentHasSizeOfInt) {
4550 DAG.getContext()->emitError(
"POWI exponent does not match sizeof(int)");
4551 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
4559 ExpandFPLibCall(
Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4560 RTLIB::POW_F128, RTLIB::POW_PPCF128,
Results);
4564 ExpandArgFPLibCall(
Node, RTLIB::LROUND_F32,
4565 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4567 RTLIB::LROUND_PPCF128,
Results);
4571 ExpandArgFPLibCall(
Node, RTLIB::LLROUND_F32,
4572 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4573 RTLIB::LLROUND_F128,
4574 RTLIB::LLROUND_PPCF128,
Results);
4578 ExpandArgFPLibCall(
Node, RTLIB::LRINT_F32,
4579 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4581 RTLIB::LRINT_PPCF128,
Results);
4585 ExpandArgFPLibCall(
Node, RTLIB::LLRINT_F32,
4586 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4588 RTLIB::LLRINT_PPCF128,
Results);
4592 ExpandFPLibCall(
Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4593 RTLIB::DIV_F80, RTLIB::DIV_F128,
4598 ExpandFPLibCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
4599 RTLIB::REM_F80, RTLIB::REM_F128,
4604 ExpandFPLibCall(
Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4605 RTLIB::FMA_F80, RTLIB::FMA_F128,
4610 ExpandFPLibCall(
Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4611 RTLIB::ADD_F80, RTLIB::ADD_F128,
4616 ExpandFPLibCall(
Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4617 RTLIB::MUL_F80, RTLIB::MUL_F128,
4621 if (
Node->getValueType(0) == MVT::f32) {
4622 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32,
Node,
false).first);
4626 if (
Node->getValueType(0) == MVT::f32) {
4628 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4629 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32,
Node->getOperand(1),
4632 Results.push_back(Tmp.second);
4636 if (
Node->getValueType(0) == MVT::f32) {
4638 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4639 DAG, RTLIB::FPEXT_F16_F32, MVT::f32,
Node->getOperand(1), CallOptions,
4642 Results.push_back(Tmp.second);
4649 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16");
4650 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4656 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_bf16");
4657 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4665 bool IsStrict =
Node->isStrictFPOpcode();
4668 EVT SVT =
Node->getOperand(IsStrict ? 1 : 0).getValueType();
4669 EVT RVT =
Node->getValueType(0);
4677 for (
unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4678 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4686 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4691 NVT,
Node->getOperand(IsStrict ? 1 : 0));
4694 std::pair<SDValue, SDValue> Tmp =
4695 TLI.makeLibCall(DAG, LC, RVT,
Op, CallOptions, dl, Chain);
4698 Results.push_back(Tmp.second);
4706 bool IsStrict =
Node->isStrictFPOpcode();
4711 EVT SVT =
Op.getValueType();
4712 EVT RVT =
Node->getValueType(0);
4720 for (
unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4721 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4729 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4733 std::pair<SDValue, SDValue> Tmp =
4734 TLI.makeLibCall(DAG, LC, NVT,
Op, CallOptions, dl, Chain);
4739 Results.push_back(Tmp.second);
4750 bool IsStrict =
Node->isStrictFPOpcode();
4753 EVT VT =
Node->getValueType(0);
4754 assert(cast<ConstantSDNode>(
Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4755 "Unable to expand as libcall if it is not normal rounding");
4758 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4761 std::pair<SDValue, SDValue> Tmp =
4762 TLI.makeLibCall(DAG, LC, VT,
Op, CallOptions,
SDLoc(
Node), Chain);
4765 Results.push_back(Tmp.second);
4771 Node->getValueType(0)),
4772 Node,
false).first);
4785 Node->getValueType(0));
4787 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4790 std::pair<SDValue, SDValue> Tmp =
4791 TLI.makeLibCall(DAG, LC,
Node->getValueType(0),
Node->getOperand(1),
4794 Results.push_back(Tmp.second);
4799 ExpandFPLibCall(
Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4800 RTLIB::SUB_F80, RTLIB::SUB_F128,
4806 RTLIB::SREM_I16, RTLIB::SREM_I32,
4807 RTLIB::SREM_I64, RTLIB::SREM_I128));
4812 RTLIB::UREM_I16, RTLIB::UREM_I32,
4813 RTLIB::UREM_I64, RTLIB::UREM_I128));
4818 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4819 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4824 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4825 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4835 RTLIB::MUL_I16, RTLIB::MUL_I32,
4836 RTLIB::MUL_I64, RTLIB::MUL_I128));
4839 switch (
Node->getSimpleValueType(0).SimpleTy) {
4843 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32,
Node,
false).first);
4846 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64,
Node,
false).first);
4849 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128,
Node,
false).first);
4856 SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
4859 DAG.makeStateFunctionCall(RTLIB::FESETENV,
Ptr, Chain, dl));
4866 DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl));
4873 DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl));
4879 EVT ModeVT =
Node->getValueType(0);
4881 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4882 SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr,
4883 Node->getOperand(0), dl);
4885 ModeVT, dl, Chain, StackPtr,
4895 EVT ModeVT =
Mode.getValueType();
4897 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4898 SDValue StInst = DAG.getStore(
4899 Node->getOperand(0), dl, Mode, StackPtr,
4902 DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl));
4910 EVT PtrTy = TLI.getPointerTy(
DL);
4912 Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
4913 Node->getOperand(0), dl));
4920 LLVM_DEBUG(
dbgs() <<
"Successfully converted node to libcall\n");
4929 MVT EltVT,
MVT NewEltVT) {
4931 MVT MidVT = OldEltsPerNewElt == 1
4938void SelectionDAGLegalize::PromoteNode(
SDNode *
Node) {
4941 MVT OVT =
Node->getSimpleValueType(0);
4947 OVT =
Node->getOperand(0).getSimpleValueType();
4953 OVT =
Node->getOperand(1).getSimpleValueType();
4956 OVT =
Node->getOperand(2).getSimpleValueType();
4957 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), OVT);
4959 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4960 switch (
Node->getOpcode()) {
4980 DAG.getConstant(TopBit, dl, NVT));
4984 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
4998 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5001 DAG.getConstant(DiffBits, dl,
5002 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
5015 Results.push_back(PromoteLegalFP_TO_INT_SAT(
Node, dl));
5032 &&
"VAARG promotion is supported only for vectors or integer types");
5037 Tmp1 = DAG.getVAArg(NVT, dl, Chain,
Ptr,
Node->getOperand(2),
5038 Node->getConstantOperandVal(3));
5041 Tmp2 = DAG.
getNode(TruncOp, dl, OVT, Tmp1);
5045 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), Tmp2);
5046 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
5048 UpdatedNodes->insert(Tmp2.
getNode());
5049 UpdatedNodes->insert(Chain.
getNode());
5066 unsigned ExtOp, TruncOp;
5073 switch (
Node->getOpcode()) {
5089 if (TLI.isSExtCheaperThanZExt(OVT, NVT))
5098 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5099 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5101 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5102 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
5110 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5111 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5114 auto &
DL = DAG.getDataLayout();
5118 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(
DL, NVT)));
5124 unsigned ExtOp, TruncOp;
5125 if (
Node->getValueType(0).isVector() ||
5129 }
else if (
Node->getValueType(0).isInteger()) {
5136 Tmp1 =
Node->getOperand(0);
5138 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5139 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5141 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5144 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1);
5146 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1,
5147 DAG.getIntPtrConstant(0, dl));
5159 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5168 Node->getOperand(2));
5176 MVT CVT =
Node->getSimpleValueType(0);
5177 assert(CVT == OVT &&
"not handled");
5185 if (TLI.isCondCodeLegal(CCCode, CVT)) {
5186 Tmp1 =
Node->getOperand(0);
5187 Tmp2 =
Node->getOperand(1);
5189 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5190 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5193 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5194 Tmp4 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5204 DAG.getIntPtrConstant(0, dl,
true));
5216 TLI.isSExtCheaperThanZExt(
Node->getOperand(0).getValueType(), NVT))
5221 if (
Node->isStrictFPOpcode()) {
5223 std::tie(Tmp1, std::ignore) =
5224 DAG.getStrictFPExtendOrRound(
Node->getOperand(1), InChain, dl, NVT);
5225 std::tie(Tmp2, std::ignore) =
5226 DAG.getStrictFPExtendOrRound(
Node->getOperand(2), InChain, dl, NVT);
5228 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
5229 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
5230 Results.push_back(DAG.getNode(
Node->getOpcode(), dl, VTs,
5231 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5236 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5237 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5239 Tmp2,
Node->getOperand(2),
Node->getFlags()));
5246 cast<CondCodeSDNode>(
Node->getOperand(1))->get();
5249 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5250 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5252 Node->getOperand(0),
Node->getOperand(1),
5253 Tmp1, Tmp2,
Node->getOperand(4)));
5268 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
5272 DAG.getIntPtrConstant(0, dl,
true)));
5283 {
Node->getOperand(0),
Node->getOperand(1)});
5285 {
Node->getOperand(0),
Node->getOperand(2)});
5288 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5289 {Tmp3, Tmp1, Tmp2});
5291 {Tmp1.
getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5301 DAG.getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5302 DAG.getIntPtrConstant(0, dl,
true)));
5306 {
Node->getOperand(0),
Node->getOperand(1)});
5308 {
Node->getOperand(0),
Node->getOperand(2)});
5310 {
Node->getOperand(0),
Node->getOperand(3)});
5313 Tmp4 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5314 {Tmp4, Tmp1, Tmp2, Tmp3});
5316 {Tmp4.
getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5324 Tmp2 =
Node->getOperand(1);
5325 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5335 DAG.getIntPtrConstant(isTrunc, dl,
true)));
5340 {
Node->getOperand(0),
Node->getOperand(1)});
5341 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5344 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5354 DAG.getIntPtrConstant(0, dl,
true)));
5379 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5382 DAG.getIntPtrConstant(0, dl,
true)));
5400 {
Node->getOperand(0),
Node->getOperand(1)});
5401 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5404 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5419 "Invalid promote type for build_vector");
5425 for (
unsigned I = 0, E =
Node->getNumOperands();
I != E; ++
I) {
5454 "Invalid promote type for extract_vector_elt");
5461 EVT IdxVT =
Idx.getValueType();
5463 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
5469 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5470 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5478 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
5500 "Invalid promote type for insert_vector_elt");
5508 EVT IdxVT =
Idx.getValueType();
5511 SDValue Factor = DAG.getConstant(NewEltsPerOldElt,
SDLoc(), IdxVT);
5518 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5519 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5523 CastVal, IdxOffset);
5526 NewVec, Elt, InEltIdx);
5565 "unexpected promotion type");
5567 "unexpected atomic_swap with illegal type");
5571 DAG.getVTList(NVT, MVT::Other),
5572 { AM->getChain(), AM->getBasePtr(), CastVal },
5580 MVT ScalarType =
Scalar.getSimpleValueType();
5584 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5589 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5592 DAG.getIntPtrConstant(0, dl,
true)));
5618 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes);
5625 bool AnyLegalized =
false;
5636 if (LegalizedNodes.
insert(
N).second) {
5637 AnyLegalized =
true;
5658 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes, &UpdatedNodes);
5665 return LegalizedNodes.
count(
N);
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
static MaybeAlign getAlign(Value *Ptr)
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI)
Return true if sincos libcall is available.
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
static MachineMemOperand * getStackAlignedMMO(SDValue StackPtr, MachineFunction &MF, bool isObjectScalable)
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This is an SDNode representing atomic operations.
const SDValue & getVal() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
const BasicBlock & back() const
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOStore
The memory access writes data.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetLowering & getTargetLoweringInfo() const
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
allnodes_const_iterator allnodes_end() const
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const DataLayout & getDataLayout() const
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
LLVMContext * getContext() const
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
Information about stack frame layout on the target.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
constexpr ScalarTy getFixedValue() const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ FRAME_TO_ARGS_OFFSET
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
@ RESET_FPENV
Set floating-point environment to default state.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ GLOBAL_OFFSET_TABLE
The address of the GOT.
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ SET_FPENV_MEM
Sets the current floating point environment.
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
@ Undef
Value of the register doesn't matter.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
int32_t ExponentType
A signed type to represent a floating point numbers unbiased exponent.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)