58#define DEBUG_TYPE "legalizedag"
64struct FloatSignAsInt {
87class SelectionDAGLegalize {
99 EVT getSetCCResultType(
EVT VT)
const {
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
166 void getSignAsIntValue(FloatSignAsInt &State,
const SDLoc &
DL,
168 SDValue modifySignAsInt(
const FloatSignAsInt &State,
const SDLoc &
DL,
215 dbgs() <<
" with: "; New->dump(&DAG));
218 "Replacing one node with another that produces a different number "
222 UpdatedNodes->
insert(New);
228 dbgs() <<
" with: "; New->dump(&DAG));
232 UpdatedNodes->
insert(New.getNode());
233 ReplacedNode(Old.getNode());
240 for (
unsigned i = 0, e = Old->
getNumValues(); i != e; ++i) {
244 UpdatedNodes->
insert(New[i].getNode());
251 dbgs() <<
" with: "; New->dump(&DAG));
255 UpdatedNodes->
insert(New.getNode());
256 ReplacedNode(Old.getNode());
266 bool isObjectScalable) {
268 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();
274 ObjectSize, MFI.getObjectAlign(FI));
281SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
286 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
288 assert(NumEltsGrowth &&
"Cannot promote to vector type with fewer elts!");
290 if (NumEltsGrowth == 1)
291 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
294 for (
unsigned i = 0; i != NumMaskElts; ++i) {
296 for (
unsigned j = 0;
j != NumEltsGrowth; ++
j) {
303 assert(NewMask.
size() == NumDestElts &&
"Non-integer NumEltsGrowth?");
304 assert(TLI.isShuffleMaskLegal(NewMask, NVT) &&
"Shuffle not legal?");
305 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
324 assert((VT == MVT::f64 || VT == MVT::f32) &&
"Invalid type expansion");
326 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
336 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
342 TLI.ShouldShrinkFPConstant(OrigVT)) {
345 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
353 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
354 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
363 OrigVT, dl, DAG.getEntryNode(), CPIdx,
371 EVT VT =
CP->getValueType(0);
372 SDValue CPIdx = DAG.getConstantPool(
CP->getConstantIntValue(),
373 TLI.getPointerTy(DAG.getDataLayout()));
374 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
376 VT, dl, DAG.getEntryNode(), CPIdx,
402 for (
unsigned i = 0; i != NumElts; ++i)
403 ShufOps.
push_back(i != InsertPos->getZExtValue() ? i : NumElts);
405 return DAG.getVectorShuffle(Vec.
getValueType(), dl, Vec, ScVec, ShufOps);
408 return ExpandInsertToVectorThroughStack(
Op);
435 TLI.isTypeLegal(MVT::i32)) {
437 bitcastToAPInt().zextOrTrunc(32),
438 SDLoc(CFP), MVT::i32);
439 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
440 ST->getOriginalAlign(), MMOFlags, AAInfo);
446 if (TLI.isTypeLegal(MVT::i64)) {
448 zextOrTrunc(64),
SDLoc(CFP), MVT::i64);
449 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
450 ST->getOriginalAlign(), MMOFlags, AAInfo);
453 if (TLI.isTypeLegal(MVT::i32) && !
ST->isVolatile()) {
460 if (DAG.getDataLayout().isBigEndian())
463 Lo = DAG.getStore(Chain, dl,
Lo,
Ptr,
ST->getPointerInfo(),
464 ST->getOriginalAlign(), MMOFlags, AAInfo);
466 Hi = DAG.getStore(Chain, dl,
Hi,
Ptr,
467 ST->getPointerInfo().getWithOffset(4),
468 ST->getOriginalAlign(), MMOFlags, AAInfo);
477void SelectionDAGLegalize::LegalizeStoreOps(
SDNode *
Node) {
486 if (!
ST->isTruncatingStore()) {
488 if (
SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
489 ReplaceNode(ST, OptStore);
494 MVT VT =
Value.getSimpleValueType();
495 switch (TLI.getOperationAction(
ISD::STORE, VT)) {
497 case TargetLowering::Legal: {
500 EVT MemVT =
ST->getMemoryVT();
502 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
503 *
ST->getMemOperand())) {
506 ReplaceNode(
SDValue(ST, 0), Result);
511 case TargetLowering::Custom: {
518 case TargetLowering::Promote: {
521 "Can only promote stores to same size type");
524 ST->getOriginalAlign(), MMOFlags, AAInfo);
534 EVT StVT =
ST->getMemoryVT();
537 auto &
DL = DAG.getDataLayout();
539 if (StWidth != StSize) {
544 Value = DAG.getZeroExtendInReg(
Value, dl, StVT);
546 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), NVT,
547 ST->getOriginalAlign(), MMOFlags, AAInfo);
553 unsigned LogStWidth =
Log2_32(StWidthBits);
555 unsigned RoundWidth = 1 << LogStWidth;
556 assert(RoundWidth < StWidthBits);
557 unsigned ExtraWidth = StWidthBits - RoundWidth;
558 assert(ExtraWidth < RoundWidth);
559 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
560 "Store size not an integral number of bytes!");
564 unsigned IncrementSize;
566 if (
DL.isLittleEndian()) {
569 Lo = DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(),
570 RoundVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
573 IncrementSize = RoundWidth / 8;
578 DAG.getConstant(RoundWidth, dl,
579 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
580 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
581 ST->getPointerInfo().getWithOffset(IncrementSize),
582 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
589 DAG.getConstant(ExtraWidth, dl,
590 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
591 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
ST->getPointerInfo(), RoundVT,
592 ST->getOriginalAlign(), MMOFlags, AAInfo);
595 IncrementSize = RoundWidth / 8;
597 DAG.getConstant(IncrementSize, dl,
598 Ptr.getValueType()));
600 ST->getPointerInfo().getWithOffset(IncrementSize),
601 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
608 switch (TLI.getTruncStoreAction(
ST->getValue().getValueType(), StVT)) {
610 case TargetLowering::Legal: {
611 EVT MemVT =
ST->getMemoryVT();
614 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
615 *
ST->getMemOperand())) {
617 ReplaceNode(
SDValue(ST, 0), Result);
621 case TargetLowering::Custom: {
627 case TargetLowering::Expand:
629 "Vector Stores are handled in LegalizeVectorOps");
634 if (TLI.isTypeLegal(StVT)) {
637 ST->getOriginalAlign(), MMOFlags, AAInfo);
642 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
645 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), StVT,
646 ST->getOriginalAlign(), MMOFlags, AAInfo);
655void SelectionDAGLegalize::LegalizeLoadOps(
SDNode *
Node) {
664 LLVM_DEBUG(
dbgs() <<
"Legalizing non-extending load operation\n");
665 MVT VT =
Node->getSimpleValueType(0);
669 switch (TLI.getOperationAction(
Node->getOpcode(), VT)) {
671 case TargetLowering::Legal: {
672 EVT MemVT =
LD->getMemoryVT();
676 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
677 *
LD->getMemOperand())) {
678 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
682 case TargetLowering::Custom:
683 if (
SDValue Res = TLI.LowerOperation(RVal, DAG)) {
689 case TargetLowering::Promote: {
690 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
692 "Can only promote loads to same size type");
694 SDValue Res = DAG.getLoad(NVT, dl, Chain,
Ptr,
LD->getMemOperand());
702 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), RVal);
703 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), RChain);
705 UpdatedNodes->insert(RVal.
getNode());
706 UpdatedNodes->insert(RChain.
getNode());
714 EVT SrcVT =
LD->getMemoryVT();
728 TLI.getLoadExtAction(ExtType,
Node->getValueType(0), MVT::i1) ==
729 TargetLowering::Promote)) {
743 Chain,
Ptr,
LD->getPointerInfo(), NVT,
744 LD->getOriginalAlign(), MMOFlags, AAInfo);
752 Result, DAG.getValueType(SrcVT));
756 Result.getValueType(), Result,
757 DAG.getValueType(SrcVT));
765 unsigned LogSrcWidth =
Log2_32(SrcWidthBits);
767 unsigned RoundWidth = 1 << LogSrcWidth;
768 assert(RoundWidth < SrcWidthBits);
769 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
770 assert(ExtraWidth < RoundWidth);
771 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
772 "Load size not an integral number of bytes!");
776 unsigned IncrementSize;
777 auto &
DL = DAG.getDataLayout();
779 if (
DL.isLittleEndian()) {
783 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
787 IncrementSize = RoundWidth / 8;
790 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
791 LD->getPointerInfo().getWithOffset(IncrementSize),
792 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
802 DAG.getConstant(RoundWidth, dl,
803 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
811 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
812 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
816 IncrementSize = RoundWidth / 8;
820 LD->getPointerInfo().getWithOffset(IncrementSize),
821 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
831 DAG.getConstant(ExtraWidth, dl,
832 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
840 bool isCustom =
false;
841 switch (TLI.getLoadExtAction(ExtType,
Node->getValueType(0),
844 case TargetLowering::Custom:
847 case TargetLowering::Legal:
859 EVT MemVT =
LD->getMemoryVT();
861 if (!TLI.allowsMemoryAccess(*DAG.getContext(),
DL, MemVT,
862 *
LD->getMemOperand())) {
863 std::tie(
Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
868 case TargetLowering::Expand: {
869 EVT DestVT =
Node->getValueType(0);
875 (TLI.isTypeLegal(SrcVT) ||
876 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
883 SrcVT,
LD->getMemOperand());
886 Value = DAG.getNode(ExtendOp, dl,
Node->getValueType(0), Load);
887 Chain =
Load.getValue(1);
896 if (SVT == MVT::f16 || SVT == MVT::bf16) {
902 Ptr, ISrcVT,
LD->getMemOperand());
906 Chain =
Result.getValue(1);
912 "Vector Loads are handled in LegalizeVectorOps");
919 "EXTLOAD should always be supported!");
923 Node->getValueType(0),
925 LD->getMemOperand());
930 Result, DAG.getValueType(SrcVT));
932 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
934 Chain =
Result.getValue(1);
945 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
947 UpdatedNodes->insert(
Value.getNode());
948 UpdatedNodes->insert(Chain.
getNode());
955void SelectionDAGLegalize::LegalizeOp(
SDNode *
Node) {
964 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
965 assert(TLI.getTypeAction(*DAG.getContext(),
Node->getValueType(i)) ==
966 TargetLowering::TypeLegal &&
967 "Unexpected illegal type!");
970 assert((TLI.getTypeAction(*DAG.getContext(),
Op.getValueType()) ==
971 TargetLowering::TypeLegal ||
974 "Unexpected illegal type!");
979 bool SimpleFinishLegalizing =
true;
980 switch (
Node->getOpcode()) {
985 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
988 Action = TLI.getOperationAction(
Node->getOpcode(),
989 Node->getValueType(0));
992 Action = TLI.getOperationAction(
Node->getOpcode(),
993 Node->getValueType(0));
994 if (Action != TargetLowering::Promote)
995 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
999 Action = TLI.getOperationAction(
Node->getOpcode(),
1000 Node->getOperand(1).getValueType());
1011 Action = TLI.getOperationAction(
Node->getOpcode(),
1012 Node->getOperand(0).getValueType());
1025 Action = TLI.getOperationAction(
Node->getOpcode(),
1026 Node->getOperand(1).getValueType());
1029 EVT InnerType = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1030 Action = TLI.getOperationAction(
Node->getOpcode(), InnerType);
1034 Action = TLI.getOperationAction(
Node->getOpcode(),
1035 Node->getOperand(1).getValueType());
1044 unsigned Opc =
Node->getOpcode();
1049 : (Opc ==
ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1051 unsigned CompareOperand = Opc ==
ISD::BR_CC ? 2
1055 MVT OpVT =
Node->getOperand(CompareOperand).getSimpleValueType();
1057 cast<CondCodeSDNode>(
Node->getOperand(CCOperand))->get();
1058 Action = TLI.getCondCodeAction(CCCode, OpVT);
1059 if (Action == TargetLowering::Legal) {
1061 Action = TLI.getOperationAction(
Node->getOpcode(),
1062 Node->getValueType(0));
1064 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
1072 SimpleFinishLegalizing =
false;
1079 SimpleFinishLegalizing =
false;
1092 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1093 if (Action == TargetLowering::Legal)
1094 Action = TargetLowering::Expand;
1104 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1105 if (Action == TargetLowering::Legal)
1106 Action = TargetLowering::Custom;
1112 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::i64);
1119 Action = TargetLowering::Legal;
1122 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1123 if (Action == TargetLowering::Expand) {
1127 Node->getOperand(0));
1134 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1135 if (Action == TargetLowering::Expand) {
1139 Node->getOperand(0));
1153 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1163 unsigned Scale =
Node->getConstantOperandVal(2);
1164 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
1165 Node->getValueType(0), Scale);
1169 Action = TLI.getOperationAction(
Node->getOpcode(),
1170 cast<MaskedScatterSDNode>(
Node)->getValue().getValueType());
1173 Action = TLI.getOperationAction(
Node->getOpcode(),
1174 cast<MaskedStoreSDNode>(
Node)->getValue().getValueType());
1176 case ISD::VP_SCATTER:
1177 Action = TLI.getOperationAction(
1179 cast<VPScatterSDNode>(
Node)->getValue().getValueType());
1182 Action = TLI.getOperationAction(
1184 cast<VPStoreSDNode>(
Node)->getValue().getValueType());
1186 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1187 Action = TLI.getOperationAction(
1189 cast<VPStridedStoreSDNode>(
Node)->getValue().getValueType());
1207 Action = TLI.getOperationAction(
1208 Node->getOpcode(),
Node->getOperand(0).getValueType());
1212 case ISD::VP_REDUCE_FADD:
1213 case ISD::VP_REDUCE_FMUL:
1214 case ISD::VP_REDUCE_ADD:
1215 case ISD::VP_REDUCE_MUL:
1216 case ISD::VP_REDUCE_AND:
1217 case ISD::VP_REDUCE_OR:
1218 case ISD::VP_REDUCE_XOR:
1219 case ISD::VP_REDUCE_SMAX:
1220 case ISD::VP_REDUCE_SMIN:
1221 case ISD::VP_REDUCE_UMAX:
1222 case ISD::VP_REDUCE_UMIN:
1223 case ISD::VP_REDUCE_FMAX:
1224 case ISD::VP_REDUCE_FMIN:
1225 case ISD::VP_REDUCE_SEQ_FADD:
1226 case ISD::VP_REDUCE_SEQ_FMUL:
1227 Action = TLI.getOperationAction(
1228 Node->getOpcode(),
Node->getOperand(1).getValueType());
1230 case ISD::VP_CTTZ_ELTS:
1231 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
1232 Action = TLI.getOperationAction(
Node->getOpcode(),
1233 Node->getOperand(0).getValueType());
1237 Action = TLI.getCustomOperationAction(*
Node);
1239 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1244 if (SimpleFinishLegalizing) {
1246 switch (
Node->getOpcode()) {
1265 NewNode = DAG.UpdateNodeOperands(
Node, Op0, SAO);
1285 NewNode = DAG.UpdateNodeOperands(
Node, Op0, Op1, SAO);
1291 if (NewNode !=
Node) {
1292 ReplaceNode(
Node, NewNode);
1296 case TargetLowering::Legal:
1299 case TargetLowering::Custom:
1307 if (
Node->getNumValues() == 1) {
1311 Node->getValueType(0) == MVT::Glue) &&
1312 "Type mismatch for custom legalized operation");
1320 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i) {
1324 Node->getValueType(i) == MVT::Glue) &&
1325 "Type mismatch for custom legalized operation");
1329 ReplaceNode(
Node, ResultVals.
data());
1334 case TargetLowering::Expand:
1335 if (ExpandNode(
Node))
1338 case TargetLowering::LibCall:
1339 ConvertNodeToLibcall(
Node);
1341 case TargetLowering::Promote:
1347 switch (
Node->getOpcode()) {
1360 return LegalizeLoadOps(
Node);
1362 return LegalizeStoreOps(
Node);
1366SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(
SDValue Op) {
1386 if (
ST->isIndexed() ||
ST->isTruncatingStore() ||
1387 ST->getValue() != Vec)
1392 if (!
ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1401 ST->hasPredecessor(
Op.getNode()))
1414 StackPtr = DAG.CreateStackTemporary(VecVT);
1417 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO);
1421 Align ElementAlignment =
1422 std::min(cast<StoreSDNode>(Ch)->
getAlign(),
1423 DAG.getDataLayout().getPrefTypeAlign(
1424 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1426 if (
Op.getValueType().isVector()) {
1427 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1428 Op.getValueType(),
Idx);
1429 NewLoad = DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr,
1432 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1433 NewLoad = DAG.getExtLoad(
ISD::EXTLOAD, dl,
Op.getValueType(), Ch, StackPtr,
1439 DAG.ReplaceAllUsesOfValueWith(Ch,
SDValue(NewLoad.
getNode(), 1));
1445 NewLoadOperands[0] = Ch;
1447 SDValue(DAG.UpdateNodeOperands(NewLoad.
getNode(), NewLoadOperands), 0);
1451SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(
SDValue Op) {
1452 assert(
Op.getValueType().isVector() &&
"Non-vector insert subvector!");
1463 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1468 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1471 Idx = DAG.getFreeze(
Idx);
1476 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, PartVT,
Idx);
1480 Ch, dl, Part, SubStackPtr,
1484 TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1487 Ch = DAG.getTruncStore(
1488 Ch, dl, Part, SubStackPtr,
1494 return DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1500 "Unexpected opcode!");
1506 EVT VT =
Node->getValueType(0);
1508 :
Node->getOperand(0).getValueType();
1510 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1511 int FI = cast<FrameIndexSDNode>(FIPtr.
getNode())->getIndex();
1518 assert(TypeByteSize > 0 &&
"Vector element type too small for stack store!");
1522 bool Truncate = isa<BuildVectorSDNode>(
Node) &&
1523 MemVT.
bitsLT(
Node->getOperand(0).getValueType());
1526 for (
unsigned i = 0, e =
Node->getNumOperands(); i != e; ++i) {
1528 if (
Node->getOperand(i).isUndef())
continue;
1530 unsigned Offset = TypeByteSize*i;
1536 Stores.
push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1540 Stores.
push_back(DAG.getStore(DAG.getEntryNode(), dl,
Node->getOperand(i),
1545 if (!Stores.
empty())
1548 StoreChain = DAG.getEntryNode();
1551 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1557void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1560 EVT FloatVT =
Value.getValueType();
1562 State.FloatVT = FloatVT;
1565 if (TLI.isTypeLegal(IVT)) {
1568 State.SignBit = NumBits - 1;
1574 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1577 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1582 State.Chain = DAG.getStore(DAG.getEntryNode(),
DL,
Value, State.FloatPtr,
1583 State.FloatPointerInfo);
1590 State.IntPointerInfo = State.FloatPointerInfo;
1593 unsigned ByteOffset = (NumBits / 8) - 1;
1600 State.IntPtr = IntPtr;
1601 State.IntValue = DAG.getExtLoad(
ISD::EXTLOAD,
DL, LoadTy, State.Chain, IntPtr,
1602 State.IntPointerInfo, MVT::i8);
1609SDValue SelectionDAGLegalize::modifySignAsInt(
const FloatSignAsInt &State,
1616 SDValue Chain = DAG.getTruncStore(State.Chain,
DL, NewIntValue, State.IntPtr,
1617 State.IntPointerInfo, MVT::i8);
1618 return DAG.getLoad(State.FloatVT,
DL, Chain, State.FloatPtr,
1619 State.FloatPointerInfo);
1628 FloatSignAsInt SignAsInt;
1629 getSignAsIntValue(SignAsInt,
DL, Sign);
1631 EVT IntVT = SignAsInt.IntValue.getValueType();
1632 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1638 if (TLI.isOperationLegalOrCustom(
ISD::FABS, FloatVT) &&
1639 TLI.isOperationLegalOrCustom(
ISD::FNEG, FloatVT)) {
1642 SDValue Cond = DAG.getSetCC(
DL, getSetCCResultType(IntVT), SignBit,
1644 return DAG.getSelect(
DL, FloatVT,
Cond, NegValue, AbsValue);
1648 FloatSignAsInt MagAsInt;
1649 getSignAsIntValue(MagAsInt,
DL, Mag);
1650 EVT MagVT = MagAsInt.IntValue.getValueType();
1651 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask,
DL, MagVT);
1656 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1657 EVT ShiftVT = IntVT;
1663 if (ShiftAmount > 0) {
1664 SDValue ShiftCnst = DAG.getConstant(ShiftAmount,
DL, ShiftVT);
1666 }
else if (ShiftAmount < 0) {
1667 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount,
DL, ShiftVT);
1677 return modifySignAsInt(MagAsInt,
DL, CopiedSign);
1683 FloatSignAsInt SignAsInt;
1684 getSignAsIntValue(SignAsInt,
DL,
Node->getOperand(0));
1685 EVT IntVT = SignAsInt.IntValue.getValueType();
1688 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1693 return modifySignAsInt(SignAsInt,
DL, SignFlip);
1701 EVT FloatVT =
Value.getValueType();
1708 FloatSignAsInt ValueAsInt;
1709 getSignAsIntValue(ValueAsInt,
DL,
Value);
1710 EVT IntVT = ValueAsInt.IntValue.getValueType();
1711 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask,
DL, IntVT);
1714 return modifySignAsInt(ValueAsInt,
DL, ClearedSign);
1717void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(
SDNode*
Node,
1719 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1720 assert(SPReg &&
"Target cannot require DYNAMIC_STACKALLOC expansion and"
1721 " not tell us which reg is the stack pointer!");
1723 EVT VT =
Node->getValueType(0);
1731 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1734 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1736 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1744 if (Alignment > StackAlign)
1746 DAG.getConstant(-Alignment.
value(), dl, VT));
1747 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);
1749 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0,
SDValue(), dl);
1761 return EmitStackConvert(
SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1769 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1772 if ((SrcVT.
bitsGT(SlotVT) &&
1773 !TLI.isTruncStoreLegalOrCustom(
SrcOp.getValueType(), SlotVT)) ||
1774 (SlotVT.
bitsLT(DestVT) &&
1775 !TLI.isLoadExtLegalOrCustom(
ISD::EXTLOAD, DestVT, SlotVT)))
1779 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1780 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1792 if (SrcVT.
bitsGT(SlotVT))
1793 Store = DAG.getTruncStore(Chain, dl,
SrcOp, FIPtr, PtrInfo,
1797 Store = DAG.getStore(Chain, dl,
SrcOp, FIPtr, PtrInfo, SrcAlign);
1801 if (SlotVT.
bitsEq(DestVT))
1802 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1805 return DAG.getExtLoad(
ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1818 SDValue Ch = DAG.getTruncStore(
1819 DAG.getEntryNode(), dl,
Node->getOperand(0), StackPtr,
1821 Node->getValueType(0).getVectorElementType());
1823 Node->getValueType(0), dl, Ch, StackPtr,
1830 unsigned NumElems =
Node->getNumOperands();
1832 EVT VT =
Node->getValueType(0);
1844 for (
unsigned i = 0; i < NumElems; ++i) {
1855 while (IntermedVals.
size() > 2) {
1856 NewIntermedVals.
clear();
1857 for (
unsigned i = 0, e = (IntermedVals.
size() & ~1u); i < e; i += 2) {
1863 FinalIndices.
reserve(IntermedVals[i].second.
size() +
1864 IntermedVals[i+1].second.
size());
1867 for (
unsigned j = 0, f = IntermedVals[i].second.
size(); j != f;
1870 FinalIndices.
push_back(IntermedVals[i].second[j]);
1872 for (
unsigned j = 0, f = IntermedVals[i+1].second.
size(); j != f;
1874 ShuffleVec[k] = NumElems + j;
1875 FinalIndices.
push_back(IntermedVals[i+1].second[j]);
1881 IntermedVals[i+1].first,
1886 std::make_pair(Shuffle, std::move(FinalIndices)));
1891 if ((IntermedVals.
size() & 1) != 0)
1894 IntermedVals.
swap(NewIntermedVals);
1898 "Invalid number of intermediate vectors");
1899 SDValue Vec1 = IntermedVals[0].first;
1901 if (IntermedVals.
size() > 1)
1902 Vec2 = IntermedVals[1].first;
1907 for (
unsigned i = 0, e = IntermedVals[0].second.
size(); i != e; ++i)
1908 ShuffleVec[IntermedVals[0].second[i]] = i;
1909 for (
unsigned i = 0, e = IntermedVals[1].second.
size(); i != e; ++i)
1910 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1924 unsigned NumElems =
Node->getNumOperands();
1927 EVT VT =
Node->getValueType(0);
1928 EVT OpVT =
Node->getOperand(0).getValueType();
1933 bool isOnlyLowElement =
true;
1934 bool MoreThanTwoValues =
false;
1936 for (
unsigned i = 0; i < NumElems; ++i) {
1941 isOnlyLowElement =
false;
1942 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1947 }
else if (!Value2.
getNode()) {
1950 }
else if (V != Value1 && V != Value2) {
1951 MoreThanTwoValues =
true;
1956 return DAG.getUNDEF(VT);
1958 if (isOnlyLowElement)
1964 for (
unsigned i = 0, e = NumElems; i !=
e; ++i) {
1966 dyn_cast<ConstantFPSDNode>(
Node->getOperand(i))) {
1969 dyn_cast<ConstantSDNode>(
Node->getOperand(i))) {
1988 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1989 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1991 VT, dl, DAG.getEntryNode(), CPIdx,
1997 for (
unsigned i = 0; i < NumElems; ++i) {
1998 if (
Node->getOperand(i).isUndef())
2003 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.
size())) {
2004 if (!MoreThanTwoValues) {
2006 for (
unsigned i = 0; i < NumElems; ++i) {
2010 ShuffleVec[i] =
V == Value1 ? 0 : NumElems;
2012 if (TLI.isShuffleMaskLegal(ShuffleVec,
Node->getValueType(0))) {
2019 Vec2 = DAG.getUNDEF(VT);
2022 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2032 return ExpandVectorBuildThroughStack(
Node);
2037 EVT VT =
Node->getValueType(0);
2040 return DAG.getSplatBuildVector(VT,
DL, SplatVal);
2051 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2052 TLI.getPointerTy(DAG.getDataLayout()));
2054 EVT RetVT =
Node->getValueType(0);
2061 SDValue InChain = DAG.getEntryNode();
2066 const Function &
F = DAG.getMachineFunction().getFunction();
2068 TLI.isInTailCallPosition(DAG,
Node, TCChain) &&
2069 (
RetTy ==
F.getReturnType() ||
F.getReturnType()->isVoidTy());
2074 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT,
isSigned);
2077 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2079 .setTailCall(isTailCall)
2080 .setSExtResult(signExtend)
2081 .setZExtResult(!signExtend)
2082 .setIsPostTypeLegalization(
true);
2084 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2089 return {DAG.getRoot(), DAG.getRoot()};
2101 EVT ArgVT =
Op.getValueType();
2105 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT,
isSigned);
2106 Entry.IsZExt = !Entry.IsSExt;
2107 Args.push_back(Entry);
2110 return ExpandLibCall(LC,
Node, std::move(Args),
isSigned);
2113void SelectionDAGLegalize::ExpandFrexpLibCall(
2116 EVT VT =
Node->getValueType(0);
2117 EVT ExpVT =
Node->getValueType(1);
2125 FPArgEntry.
Node = FPOp;
2126 FPArgEntry.
Ty = ArgTy;
2128 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2130 PtrArgEntry.
Node = StackSlot;
2131 PtrArgEntry.
Ty = PointerType::get(*DAG.getContext(),
2132 DAG.getDataLayout().getAllocaAddrSpace());
2137 auto [
Call, Chain] = ExpandLibCall(LC,
Node, std::move(Args),
false);
2141 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2145 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2147 LoadExp.
getValue(1), DAG.getRoot());
2148 DAG.setRoot(OutputChain);
2154void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2157 if (LC == RTLIB::UNKNOWN_LIBCALL)
2160 if (
Node->isStrictFPOpcode()) {
2161 EVT RetVT =
Node->getValueType(0);
2165 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2168 Node->getOperand(0));
2170 Results.push_back(Tmp.second);
2172 SDValue Tmp = ExpandLibCall(LC,
Node,
false).first;
2178void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2186 Call_F32, Call_F64, Call_F80,
2187 Call_F128, Call_PPCF128);
2198 switch (
Node->getSimpleValueType(0).SimpleTy) {
2200 case MVT::i8: LC = Call_I8;
break;
2201 case MVT::i16: LC = Call_I16;
break;
2202 case MVT::i32: LC = Call_I32;
break;
2203 case MVT::i64: LC = Call_I64;
break;
2204 case MVT::i128: LC = Call_I128;
break;
2211void SelectionDAGLegalize::ExpandArgFPLibCall(
SDNode*
Node,
2218 EVT InVT =
Node->getOperand(
Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2220 Call_F32, Call_F64, Call_F80,
2221 Call_F128, Call_PPCF128);
2227SelectionDAGLegalize::ExpandDivRemLibCall(
SDNode *
Node,
2229 unsigned Opcode =
Node->getOpcode();
2233 switch (
Node->getSimpleValueType(0).SimpleTy) {
2235 case MVT::i8: LC=
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
2236 case MVT::i16: LC=
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
2237 case MVT::i32: LC=
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
2238 case MVT::i64: LC=
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
2239 case MVT::i128: LC=
isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
2245 SDValue InChain = DAG.getEntryNode();
2247 EVT RetVT =
Node->getValueType(0);
2253 EVT ArgVT =
Op.getValueType();
2259 Args.push_back(Entry);
2263 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2265 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2268 Args.push_back(Entry);
2270 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2271 TLI.getPointerTy(DAG.getDataLayout()));
2277 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2282 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2294 switch (
Node->getSimpleValueType(0).SimpleTy) {
2296 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2297 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2298 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2299 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2300 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2323SelectionDAGLegalize::ExpandSinCosLibCall(
SDNode *
Node,
2326 switch (
Node->getSimpleValueType(0).SimpleTy) {
2328 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2329 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2330 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2331 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2332 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2338 SDValue InChain = DAG.getEntryNode();
2340 EVT RetVT =
Node->getValueType(0);
2347 Entry.Node =
Node->getOperand(0);
2349 Entry.IsSExt =
false;
2350 Entry.IsZExt =
false;
2351 Args.push_back(Entry);
2354 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2355 Entry.Node = SinPtr;
2356 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2357 Entry.IsSExt =
false;
2358 Entry.IsZExt =
false;
2359 Args.push_back(Entry);
2362 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2363 Entry.Node = CosPtr;
2364 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2365 Entry.IsSExt =
false;
2366 Entry.IsZExt =
false;
2367 Args.push_back(Entry);
2369 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2370 TLI.getPointerTy(DAG.getDataLayout()));
2374 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2375 TLI.getLibcallCallingConv(LC),
Type::getVoidTy(*DAG.getContext()), Callee,
2378 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2388 EVT VT =
Node->getValueType(0);
2391 EVT ExpVT =
N.getValueType();
2393 if (AsIntVT ==
EVT())
2406 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2411 const int Precision = APFloat::semanticsPrecision(FltSem);
2413 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2414 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2416 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2418 const APFloat One(FltSem,
"1.0");
2419 APFloat ScaleUpK =
scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2423 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2432 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2438 DAG.getSetCC(dl, SetCCVT,
N, DoubleMaxExp,
ISD::SETUGT);
2440 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2452 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2453 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2458 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2463 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2467 SDValue ScaleDownTwice = DAG.getSetCC(
2468 dl, SetCCVT,
N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2480 DAG.getNode(
ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small,
X));
2484 DAG.getNode(
ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small,
N));
2489 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2490 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2493 ExponentShiftAmt, NUW_NSW);
2495 return DAG.getNode(
ISD::FMUL, dl, VT, NewX, AsFP);
2502 EVT ExpVT =
Node->getValueType(1);
2504 if (AsIntVT ==
EVT())
2509 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2526 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2530 SDValue SmallestNormalizedInt = DAG.getConstant(
2536 DAG.getConstant(
APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2542 FractSignMaskVal.
setBit(BitSize - 1);
2545 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2547 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2549 const APFloat One(FltSem,
"1.0");
2553 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2555 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2559 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2565 SDValue AddNegSmallestNormal =
2567 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2571 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt,
ISD::SETULT);
2573 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2588 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2591 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2594 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2601 SDValue FPHalf = DAG.getConstant(
Half.bitcastToAPInt(), dl, AsIntVT);
2614 return DAG.getMergeValues({Result0, Result1}, dl);
2625 EVT DestVT =
Node->getValueType(0);
2627 unsigned OpNo =
Node->isStrictFPOpcode() ? 1 : 0;
2633 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2634 (DestVT.
bitsLE(MVT::f64) ||
2638 LLVM_DEBUG(
dbgs() <<
"32-bit [signed|unsigned] integer to float/double "
2642 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2649 DAG.getConstant(0x80000000u, dl, MVT::i32));
2652 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2655 if (DAG.getDataLayout().isBigEndian())
2658 SDValue MemChain = DAG.getEntryNode();
2661 SDValue Store1 = DAG.getStore(MemChain, dl,
Lo, StackSlot,
2674 SDValue Bias = DAG.getConstantFP(
2675 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2676 : llvm::bit_cast<double>(0x4330000000000000ULL),
2681 if (
Node->isStrictFPOpcode()) {
2683 {
Node->getOperand(0), Load, Bias});
2686 std::pair<SDValue, SDValue> ResultPair;
2688 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2689 Result = ResultPair.first;
2690 Chain = ResultPair.second;
2696 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2705 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2706 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2707 LLVM_DEBUG(
dbgs() <<
"Converting unsigned i32/i64 to f32/f64\n");
2722 EVT SetCCVT = getSetCCResultType(SrcVT);
2724 SDValue SignBitTest = DAG.getSetCC(
2725 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT),
ISD::SETLT);
2727 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2728 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2730 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2735 if (
Node->isStrictFPOpcode()) {
2738 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest,
Or, Op0);
2740 {
Node->getOperand(0), InCvt });
2748 Flags.setNoFPExcept(
Node->getFlags().hasNoFPExcept());
2749 Fast->setFlags(Flags);
2750 Flags.setNoFPExcept(
true);
2758 return DAG.getSelect(dl, DestVT, SignBitTest, Slow,
Fast);
2762 if (!TLI.isOperationLegalOrCustom(
2769 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2771 "Cannot perform lossless SINT_TO_FP!");
2774 if (
Node->isStrictFPOpcode()) {
2776 {
Node->getOperand(0), Op0 });
2780 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2783 Four = DAG.getIntPtrConstant(4, dl);
2784 SDValue CstOffset = DAG.getSelect(dl,
Zero.getValueType(),
2785 SignSet, Four, Zero);
2794 case MVT::i8 : FF = 0x43800000ULL;
break;
2795 case MVT::i16: FF = 0x47800000ULL;
break;
2796 case MVT::i32: FF = 0x4F800000ULL;
break;
2797 case MVT::i64: FF = 0x5F800000ULL;
break;
2799 if (DAG.getDataLayout().isLittleEndian())
2801 Constant *FudgeFactor = ConstantInt::get(
2805 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2806 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2810 if (DestVT == MVT::f32)
2811 FudgeInReg = DAG.getLoad(
2812 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2821 LegalizeOp(
Load.getNode());
2825 if (
Node->isStrictFPOpcode()) {
2827 { Tmp1.
getValue(1), Tmp1, FudgeInReg });
2828 Chain =
Result.getValue(1);
2832 return DAG.getNode(
ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2840void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2842 bool IsStrict =
N->isStrictFPOpcode();
2845 EVT DestVT =
N->getValueType(0);
2846 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2853 unsigned OpToUse = 0;
2861 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2869 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2881 DAG.
getNode(OpToUse, dl, {DestVT, MVT::Other},
2884 dl, NewInTy, LegalOp)});
2891 DAG.getNode(OpToUse, dl, DestVT,
2893 dl, NewInTy, LegalOp)));
2901void SelectionDAGLegalize::PromoteLegalFP_TO_INT(
SDNode *
N,
const SDLoc &dl,
2903 bool IsStrict =
N->isStrictFPOpcode();
2906 EVT DestVT =
N->getValueType(0);
2907 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2909 EVT NewOutTy = DestVT;
2911 unsigned OpToUse = 0;
2921 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2926 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2935 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2936 Operation = DAG.getNode(OpToUse, dl, VTs,
N->getOperand(0), LegalOp);
2938 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2953 unsigned Opcode =
Node->getOpcode();
2956 EVT NewOutTy =
Node->getValueType(0);
2961 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2968 Node->getOperand(1));
2974 EVT VT =
Op.getValueType();
2975 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2980 if (TLI.isOperationLegalOrPromote(
ISD::CTPOP, VT)) {
2986 DAG.getConstant(1ULL << (--i), dl, ShVT));
2991 return DAG.getNode(
ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2995 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
2996 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
2997 MVT ScalarVT =
Node->getSimpleValueType(0);
3005 assert(
Node->getOperand(0).getValueType().isFloatingPoint() &&
3006 "Only FP promotion is supported");
3010 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
3011 if (
Node->getOperand(j).getValueType().isVector() &&
3016 assert(
Node->getOperand(j).getValueType().isFloatingPoint() &&
3017 "Only FP promotion is supported");
3029 DAG.getIntPtrConstant(0,
DL,
true));
3032bool SelectionDAGLegalize::ExpandNode(
SDNode *
Node) {
3036 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3038 switch (
Node->getOpcode()) {
3040 if ((Tmp1 = TLI.expandABS(
Node, DAG)))
3045 if ((Tmp1 = TLI.expandABD(
Node, DAG)))
3049 if ((Tmp1 = TLI.expandCTPOP(
Node, DAG)))
3054 if ((Tmp1 = TLI.expandCTLZ(
Node, DAG)))
3059 if ((Tmp1 = TLI.expandCTTZ(
Node, DAG)))
3063 if ((Tmp1 = TLI.expandBITREVERSE(
Node, DAG)))
3067 if ((Tmp1 = TLI.expandBSWAP(
Node, DAG)))
3071 Results.push_back(ExpandPARITY(
Node->getOperand(0), dl));
3076 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3079 SDValue CfaArg = DAG.getSExtOrTrunc(
Node->getOperand(0), dl,
3080 TLI.getPointerTy(DAG.getDataLayout()));
3088 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3094 Results.push_back(DAG.getConstant(1, dl,
Node->getValueType(0)));
3111 DAG.getConstant(0, dl,
Node->getValueType(0)));
3117 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3123 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3124 SDValue Swap = DAG.getAtomicCmpSwap(
3126 Node->getOperand(0),
Node->getOperand(1), Zero, Zero,
3127 cast<AtomicSDNode>(
Node)->getMemOperand());
3136 Node->getOperand(0),
Node->getOperand(2),
Node->getOperand(1),
3137 cast<AtomicSDNode>(
Node)->getMemOperand());
3145 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3146 SDValue Res = DAG.getAtomicCmpSwap(
3148 Node->getOperand(0),
Node->getOperand(1),
Node->getOperand(2),
3149 Node->getOperand(3), cast<MemSDNode>(
Node)->getMemOperand());
3155 EVT AtomicType = cast<AtomicSDNode>(
Node)->getMemoryVT();
3156 EVT OuterType =
Node->getValueType(0);
3157 switch (TLI.getExtendForAtomicOps()) {
3160 DAG.getValueType(AtomicType));
3162 Node->getOperand(2), DAG.getValueType(AtomicType));
3167 DAG.getValueType(AtomicType));
3168 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3172 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3173 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3189 EVT VT =
Node->getValueType(0);
3193 cast<VTSDNode>(
RHS->getOperand(1))->getVT() == AN->
getMemoryVT())
3194 RHS =
RHS->getOperand(0);
3198 Node->getOperand(0),
Node->getOperand(1),
3208 for (
unsigned i = 0; i <
Node->getNumValues(); i++)
3212 EVT VT =
Node->getValueType(0);
3214 Results.push_back(DAG.getConstant(0, dl, VT));
3217 Results.push_back(DAG.getConstantFP(0, dl, VT));
3224 if (TLI.isStrictFPEnabled())
3228 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3229 Node->getValueType(0))
3230 == TargetLowering::Legal)
3234 if ((Tmp1 = EmitStackConvert(
Node->getOperand(1),
Node->getValueType(0),
3235 Node->getValueType(0), dl,
3236 Node->getOperand(0)))) {
3238 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_ROUND node\n");
3243 if ((Tmp1 = TLI.expandFP_ROUND(
Node, DAG))) {
3251 if ((Tmp1 = EmitStackConvert(
Node->getOperand(0),
Node->getValueType(0),
3252 Node->getValueType(0), dl)))
3258 if (TLI.isStrictFPEnabled())
3262 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3263 Node->getValueType(0))
3264 == TargetLowering::Legal)
3268 if ((Tmp1 = EmitStackConvert(
3269 Node->getOperand(1),
Node->getOperand(1).getValueType(),
3270 Node->getValueType(0), dl,
Node->getOperand(0)))) {
3272 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_EXTEND node\n");
3278 EVT SrcVT =
Op.getValueType();
3279 EVT DstVT =
Node->getValueType(0);
3285 if ((Tmp1 = EmitStackConvert(
Op, SrcVT, DstVT, dl)))
3295 if (
Op.getValueType() == MVT::bf16) {
3299 Op = DAG.getAnyExtOrTrunc(
Op, dl, MVT::i32);
3303 DAG.getConstant(16, dl,
3304 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3307 if (
Node->getValueType(0) != MVT::f32)
3314 if (
Op.getValueType() != MVT::f32)
3316 DAG.getIntPtrConstant(0, dl,
true));
3318 if (!DAG.isKnownNeverSNaN(
Op)) {
3323 DAG.getConstant(16, dl,
3324 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3327 if (
Node->getValueType(0) == MVT::bf16) {
3331 Op = DAG.getAnyExtOrTrunc(
Op, dl,
Node->getValueType(0));
3337 EVT ExtraVT = cast<VTSDNode>(
Node->getOperand(1))->getVT();
3338 EVT VT =
Node->getValueType(0);
3348 SDValue One = DAG.getConstant(1, dl, VT);
3358 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3361 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3363 Node->getOperand(0), ShiftCst);
3370 if (TLI.expandUINT_TO_FP(
Node, Tmp1, Tmp2, DAG)) {
3372 if (
Node->isStrictFPOpcode())
3379 if ((Tmp1 = ExpandLegalINT_TO_FP(
Node, Tmp2))) {
3381 if (
Node->isStrictFPOpcode())
3386 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG))
3390 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG)) {
3392 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_SINT node\n");
3397 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG))
3401 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG)) {
3403 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node,1), Tmp2);
3406 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_UINT node\n");
3412 Results.push_back(TLI.expandFP_TO_INT_SAT(
Node, DAG));
3422 if (
Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3425 Node->getOperand(0));
3427 Tmp1 = ExpandExtractFromVectorThroughStack(
SDValue(
Node, 0));
3437 Results.push_back(ExpandVectorBuildThroughStack(
Node));
3449 EVT VT =
Node->getValueType(0);
3453 if (!TLI.isTypeLegal(EltVT)) {
3454 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3459 if (NewEltVT.
bitsLT(EltVT)) {
3475 unsigned int factor =
3483 for (
unsigned fi = 0; fi < factor; ++fi)
3487 for (
unsigned fi = 0; fi < factor; ++fi)
3498 for (
unsigned i = 0; i != NumElems; ++i) {
3506 DAG.getVectorIdxConstant(
Idx, dl)));
3510 DAG.getVectorIdxConstant(
Idx - NumElems, dl)));
3513 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3520 Results.push_back(TLI.expandVectorSplice(
Node, DAG));
3524 EVT OpTy =
Node->getOperand(0).getValueType();
3525 if (
Node->getConstantOperandVal(1)) {
3529 TLI.getShiftAmountTy(
3530 Node->getOperand(0).getValueType(),
3531 DAG.getDataLayout())));
3536 Node->getOperand(0));
3544 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3545 Results.push_back(DAG.getCopyFromReg(
Node->getOperand(0), dl, SP,
3546 Node->getValueType(0)));
3549 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
3556 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3557 Results.push_back(DAG.getCopyToReg(
Node->getOperand(0), dl, SP,
3558 Node->getOperand(1)));
3564 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3579 TLI.expandIS_FPCLASS(
Node->getValueType(0),
Node->getOperand(0),
3590 switch (
Node->getOpcode()) {
3597 Tmp1 =
Node->getOperand(0);
3598 Tmp2 =
Node->getOperand(1);
3599 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3605 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG))
3611 if (
SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(
Node, DAG))
3617 EVT VT =
Node->getValueType(0);
3623 SDVTList VTs = DAG.getVTList(VT, VT);
3633 EVT VT =
Node->getValueType(0);
3637 if (TLI.getLibcallName(LC))
3643 Results.push_back(Expanded.getValue(1));
3652 if (TLI.getLibcallName(LC))
3657 Results.push_back(Expanded.getValue(1));
3665 if (
Node->getValueType(0) != MVT::f32) {
3677 if (
Node->getValueType(0) != MVT::f32) {
3682 {Node->getOperand(0), Node->getOperand(1)});
3684 {
Node->getValueType(0), MVT::Other},
3692 if (!TLI.useSoftFloat() &&
TM.Options.UnsafeFPMath) {
3694 MVT SVT =
Op.getSimpleValueType();
3695 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3701 DAG.getIntPtrConstant(0, dl,
true));
3712 DAG.shouldOptForSize()))
3713 Results.push_back(ExpandConstantFP(CFP,
true));
3718 Results.push_back(ExpandConstant(CP));
3722 EVT VT =
Node->getValueType(0);
3723 if (TLI.isOperationLegalOrCustom(
ISD::FADD, VT) &&
3724 TLI.isOperationLegalOrCustom(
ISD::FNEG, VT)) {
3733 EVT VT =
Node->getValueType(0);
3735 TLI.isOperationLegalOrCustom(
ISD::XOR, VT) &&
3736 "Don't know how to expand this subtraction!");
3737 Tmp1 = DAG.getNOT(dl,
Node->getOperand(1), VT);
3738 Tmp1 = DAG.
getNode(
ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3744 if (TLI.expandREM(
Node, Tmp1, DAG))
3751 EVT VT =
Node->getValueType(0);
3752 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3753 SDVTList VTs = DAG.getVTList(VT, VT);
3754 Tmp1 = DAG.
getNode(DivRemOpc, dl, VTs,
Node->getOperand(0),
3755 Node->getOperand(1));
3762 unsigned ExpandOpcode =
3764 EVT VT =
Node->getValueType(0);
3765 SDVTList VTs = DAG.getVTList(VT, VT);
3767 Tmp1 = DAG.
getNode(ExpandOpcode, dl, VTs,
Node->getOperand(0),
3768 Node->getOperand(1));
3776 MVT VT =
LHS.getSimpleValueType();
3777 unsigned MULHOpcode =
3780 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3782 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3788 assert(TLI.isTypeLegal(HalfType));
3789 if (TLI.expandMUL_LOHI(
Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3791 TargetLowering::MulExpansionKind::Always)) {
3792 for (
unsigned i = 0; i < 2; ++i) {
3795 SDValue Shift = DAG.getConstant(
3797 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3806 EVT VT =
Node->getValueType(0);
3807 SDVTList VTs = DAG.getVTList(VT, VT);
3813 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::SMUL_LOHI, VT);
3814 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::UMUL_LOHI, VT);
3815 bool HasMULHS = TLI.isOperationLegalOrCustom(
ISD::MULHS, VT);
3816 bool HasMULHU = TLI.isOperationLegalOrCustom(
ISD::MULHU, VT);
3817 unsigned OpToUse = 0;
3818 if (HasSMUL_LOHI && !HasMULHS) {
3820 }
else if (HasUMUL_LOHI && !HasMULHU) {
3822 }
else if (HasSMUL_LOHI) {
3824 }
else if (HasUMUL_LOHI) {
3828 Results.push_back(DAG.getNode(OpToUse, dl, VTs,
Node->getOperand(0),
3829 Node->getOperand(1)));
3837 TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
3838 TLI.isOperationLegalOrCustom(
ISD::OR, VT) &&
3839 TLI.expandMUL(
Node,
Lo,
Hi, HalfType, DAG,
3840 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3845 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3853 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG))
3858 if (
SDValue Expanded = TLI.expandROT(
Node,
true , DAG))
3865 Results.push_back(TLI.expandAddSubSat(
Node, DAG));
3875 Results.push_back(TLI.expandFixedPointMul(
Node, DAG));
3882 Node->getOperand(0),
3883 Node->getOperand(1),
3884 Node->getConstantOperandVal(2),
3907 EVT VT =
LHS.getValueType();
3911 EVT CarryType =
Node->getValueType(1);
3912 EVT SetCCType = getSetCCResultType(
Node->getValueType(0));
3914 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS,
CC);
3917 SDValue One = DAG.getConstant(1, dl, VT);
3919 DAG.
getNode(
ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3928 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero,
ISD::SETEQ)
3929 : DAG.getSetCC(dl, SetCCType, Sum, Zero,
ISD::SETEQ);
3931 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3937 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3943 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
3951 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
3959 if (TLI.expandMULO(
Node, Result, Overflow, DAG)) {
3971 DAG.getConstant(
PairTy.getSizeInBits() / 2, dl,
3972 TLI.getShiftAmountTy(
PairTy, DAG.getDataLayout())));
3977 Tmp1 =
Node->getOperand(0);
3978 Tmp2 =
Node->getOperand(1);
3979 Tmp3 =
Node->getOperand(2);
3983 cast<CondCodeSDNode>(Tmp1.
getOperand(2))->get());
3985 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3996 int JTI = cast<JumpTableSDNode>(Table.
getNode())->getIndex();
3999 EVT PTy = TLI.getPointerTy(TD);
4001 unsigned EntrySize =
4002 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4009 Index = DAG.getNode(
4014 DAG.getConstant(EntrySize, dl,
Index.getValueType()));
4023 if (TLI.isJumpTableRelative()) {
4028 TLI.getPICJumpTableRelocBase(Table, DAG));
4031 Tmp1 = TLI.expandIndirectJTBranch(dl,
LD.getValue(1),
Addr, JTI, DAG);
4038 Tmp1 =
Node->getOperand(0);
4039 Tmp2 =
Node->getOperand(1);
4045 Node->getOperand(2));
4057 Node->getOperand(2));
4065 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
4070 unsigned Offset = IsStrict ? 1 : 0;
4079 bool Legalized = TLI.LegalizeSetCCCondCode(
4080 DAG,
Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4081 Chain, IsSignaling);
4089 {Chain, Tmp1, Tmp2, Tmp3},
Node->getFlags());
4093 {Tmp1, Tmp2, Tmp3, Mask, EVL},
Node->getFlags());
4095 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl,
Node->getValueType(0), Tmp1,
4096 Tmp2, Tmp3,
Node->getFlags());
4104 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->
getValueType(0));
4107 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->
getValueType(0));
4119 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
4124 EVT VT =
Node->getValueType(0);
4127 DAG.getBoolConstant(
true, dl, VT, Tmp1VT),
4128 DAG.getBoolConstant(
false, dl, VT, Tmp1VT), Tmp3);
4135 Tmp1 =
Node->getOperand(0);
4136 Tmp2 =
Node->getOperand(1);
4137 Tmp3 =
Node->getOperand(2);
4138 Tmp4 =
Node->getOperand(3);
4139 EVT VT =
Node->getValueType(0);
4149 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4151 EVT CCVT = getSetCCResultType(CmpVT);
4153 Results.push_back(DAG.getSelect(dl, VT,
Cond, Tmp3, Tmp4));
4158 bool Legalized =
false;
4166 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4176 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4182 Legalized = TLI.LegalizeSetCCCondCode(
4186 assert(Legalized &&
"Can't legalize SELECT_CC with legal condition!");
4197 Tmp1, Tmp2, Tmp3, Tmp4,
CC);
4202 Tmp2, Tmp3, Tmp4,
CC);
4212 Tmp1 =
Node->getOperand(0);
4213 Tmp2 =
Node->getOperand(2);
4214 Tmp3 =
Node->getOperand(3);
4215 Tmp4 =
Node->getOperand(1);
4217 bool Legalized = TLI.LegalizeSetCCCondCode(
4218 DAG, getSetCCResultType(Tmp2.
getValueType()), Tmp2, Tmp3, Tmp4,
4221 assert(Legalized &&
"Can't legalize BR_CC with legal condition!");
4226 assert(!NeedInvert &&
"Don't know how to invert BR_CC!");
4229 Tmp4, Tmp2, Tmp3,
Node->getOperand(4));
4234 Tmp2, Tmp3,
Node->getOperand(4));
4249 EVT VT =
Node->getValueType(0);
4255 for (
unsigned Idx = 0;
Idx < NumElem;
Idx++) {
4258 Node->getOperand(0), DAG.getVectorIdxConstant(
Idx, dl));
4261 Node->getOperand(1), DAG.getVectorIdxConstant(
Idx, dl));
4285 Results.push_back(TLI.expandVecReduce(
Node, DAG));
4287 case ISD::VP_CTTZ_ELTS:
4288 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
4289 Results.push_back(TLI.expandVPCTTZElements(
Node, DAG));
4306 if (!TLI.isStrictFPEnabled() &&
Results.
empty() &&
Node->isStrictFPOpcode()) {
4312 switch (
Node->getOpcode()) {
4314 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4315 Node->getValueType(0))
4316 == TargetLowering::Legal)
4320 if (TLI.getStrictFPOperationAction(
4323 if (TLI.getStrictFPOperationAction(
4327 EVT VT =
Node->getValueType(0);
4331 {Node->getOperand(0), Node->getOperand(1), Neg},
4346 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4347 Node->getOperand(1).getValueType())
4348 == TargetLowering::Legal)
4365void SelectionDAGLegalize::ConvertNodeToLibcall(
SDNode *
Node) {
4370 unsigned Opc =
Node->getOpcode();
4379 .setChain(
Node->getOperand(0))
4382 DAG.getExternalSymbol(
"__sync_synchronize",
4383 TLI.getPointerTy(DAG.getDataLayout())),
4386 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4388 Results.push_back(CallResult.second);
4410 EVT RetVT =
Node->getValueType(0);
4413 if (TLI.getLibcallName(LC)) {
4420 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4421 "Unexpected atomic op or value type!");
4425 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4428 Node->getOperand(0));
4430 Results.push_back(Tmp.second);
4438 .setChain(
Node->getOperand(0))
4440 DAG.getExternalSymbol(
4441 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4443 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4445 Results.push_back(CallResult.second);
4450 ExpandFPLibCall(
Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4451 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4452 RTLIB::FMIN_PPCF128,
Results);
4459 ExpandFPLibCall(
Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4460 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4461 RTLIB::FMAX_PPCF128,
Results);
4465 ExpandFPLibCall(
Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4466 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4467 RTLIB::SQRT_PPCF128,
Results);
4470 ExpandFPLibCall(
Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4471 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4472 RTLIB::CBRT_PPCF128,
Results);
4476 ExpandFPLibCall(
Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4477 RTLIB::SIN_F80, RTLIB::SIN_F128,
4482 ExpandFPLibCall(
Node, RTLIB::COS_F32, RTLIB::COS_F64,
4483 RTLIB::COS_F80, RTLIB::COS_F128,
4492 ExpandFPLibCall(
Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4493 RTLIB::LOG_F128, RTLIB::LOG_PPCF128,
Results);
4497 ExpandFPLibCall(
Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4498 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128,
Results);
4502 ExpandFPLibCall(
Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4503 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128,
Results);
4507 ExpandFPLibCall(
Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4508 RTLIB::EXP_F128, RTLIB::EXP_PPCF128,
Results);
4512 ExpandFPLibCall(
Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4513 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128,
Results);
4516 ExpandFPLibCall(
Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4517 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128,
Results);
4521 ExpandFPLibCall(
Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4522 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4523 RTLIB::TRUNC_PPCF128,
Results);
4527 ExpandFPLibCall(
Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4528 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4529 RTLIB::FLOOR_PPCF128,
Results);
4533 ExpandFPLibCall(
Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4534 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4535 RTLIB::CEIL_PPCF128,
Results);
4539 ExpandFPLibCall(
Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4540 RTLIB::RINT_F80, RTLIB::RINT_F128,
4541 RTLIB::RINT_PPCF128,
Results);
4545 ExpandFPLibCall(
Node, RTLIB::NEARBYINT_F32,
4546 RTLIB::NEARBYINT_F64,
4547 RTLIB::NEARBYINT_F80,
4548 RTLIB::NEARBYINT_F128,
4549 RTLIB::NEARBYINT_PPCF128,
Results);
4553 ExpandFPLibCall(
Node, RTLIB::ROUND_F32,
4557 RTLIB::ROUND_PPCF128,
Results);
4561 ExpandFPLibCall(
Node, RTLIB::ROUNDEVEN_F32,
4562 RTLIB::ROUNDEVEN_F64,
4563 RTLIB::ROUNDEVEN_F80,
4564 RTLIB::ROUNDEVEN_F128,
4565 RTLIB::ROUNDEVEN_PPCF128,
Results);
4569 ExpandFPLibCall(
Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4570 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128,
Results);
4579 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
4580 if (!TLI.getLibcallName(LC)) {
4582 if (
Node->isStrictFPOpcode()) {
4585 {
Node->getValueType(0),
Node->getValueType(1)},
4586 {
Node->getOperand(0),
Node->getOperand(2)});
4589 {
Node->getValueType(0),
Node->getValueType(1)},
4596 Node->getOperand(1));
4598 Node->getValueType(0),
4603 unsigned Offset =
Node->isStrictFPOpcode() ? 1 : 0;
4604 bool ExponentHasSizeOfInt =
4605 DAG.getLibInfo().getIntSize() ==
4606 Node->getOperand(1 +
Offset).getValueType().getSizeInBits();
4607 if (!ExponentHasSizeOfInt) {
4610 DAG.getContext()->emitError(
"POWI exponent does not match sizeof(int)");
4611 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
4619 ExpandFPLibCall(
Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4620 RTLIB::POW_F128, RTLIB::POW_PPCF128,
Results);
4624 ExpandArgFPLibCall(
Node, RTLIB::LROUND_F32,
4625 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4627 RTLIB::LROUND_PPCF128,
Results);
4631 ExpandArgFPLibCall(
Node, RTLIB::LLROUND_F32,
4632 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4633 RTLIB::LLROUND_F128,
4634 RTLIB::LLROUND_PPCF128,
Results);
4638 ExpandArgFPLibCall(
Node, RTLIB::LRINT_F32,
4639 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4641 RTLIB::LRINT_PPCF128,
Results);
4645 ExpandArgFPLibCall(
Node, RTLIB::LLRINT_F32,
4646 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4648 RTLIB::LLRINT_PPCF128,
Results);
4652 ExpandFPLibCall(
Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4653 RTLIB::DIV_F80, RTLIB::DIV_F128,
4658 ExpandFPLibCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
4659 RTLIB::REM_F80, RTLIB::REM_F128,
4664 ExpandFPLibCall(
Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4665 RTLIB::FMA_F80, RTLIB::FMA_F128,
4670 ExpandFPLibCall(
Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4671 RTLIB::ADD_F80, RTLIB::ADD_F128,
4676 ExpandFPLibCall(
Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4677 RTLIB::MUL_F80, RTLIB::MUL_F128,
4681 if (
Node->getValueType(0) == MVT::f32) {
4682 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32,
Node,
false).first);
4686 if (
Node->getValueType(0) == MVT::f32) {
4688 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4689 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32,
Node->getOperand(1),
4692 Results.push_back(Tmp.second);
4696 if (
Node->getValueType(0) == MVT::f32) {
4698 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4699 DAG, RTLIB::FPEXT_F16_F32, MVT::f32,
Node->getOperand(1), CallOptions,
4702 Results.push_back(Tmp.second);
4709 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16");
4710 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4716 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_bf16");
4717 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4725 bool IsStrict =
Node->isStrictFPOpcode();
4728 EVT SVT =
Node->getOperand(IsStrict ? 1 : 0).getValueType();
4729 EVT RVT =
Node->getValueType(0);
4737 for (
unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4738 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4746 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4751 NVT,
Node->getOperand(IsStrict ? 1 : 0));
4754 std::pair<SDValue, SDValue> Tmp =
4755 TLI.makeLibCall(DAG, LC, RVT,
Op, CallOptions, dl, Chain);
4758 Results.push_back(Tmp.second);
4766 bool IsStrict =
Node->isStrictFPOpcode();
4771 EVT SVT =
Op.getValueType();
4772 EVT RVT =
Node->getValueType(0);
4780 for (
unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4781 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4789 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4793 std::pair<SDValue, SDValue> Tmp =
4794 TLI.makeLibCall(DAG, LC, NVT,
Op, CallOptions, dl, Chain);
4799 Results.push_back(Tmp.second);
4810 bool IsStrict =
Node->isStrictFPOpcode();
4813 EVT VT =
Node->getValueType(0);
4814 assert(cast<ConstantSDNode>(
Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4815 "Unable to expand as libcall if it is not normal rounding");
4818 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4821 std::pair<SDValue, SDValue> Tmp =
4822 TLI.makeLibCall(DAG, LC, VT,
Op, CallOptions,
SDLoc(
Node), Chain);
4825 Results.push_back(Tmp.second);
4831 Node->getValueType(0)),
4832 Node,
false).first);
4845 Node->getValueType(0));
4847 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4850 std::pair<SDValue, SDValue> Tmp =
4851 TLI.makeLibCall(DAG, LC,
Node->getValueType(0),
Node->getOperand(1),
4854 Results.push_back(Tmp.second);
4859 ExpandFPLibCall(
Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4860 RTLIB::SUB_F80, RTLIB::SUB_F128,
4866 RTLIB::SREM_I16, RTLIB::SREM_I32,
4867 RTLIB::SREM_I64, RTLIB::SREM_I128));
4872 RTLIB::UREM_I16, RTLIB::UREM_I32,
4873 RTLIB::UREM_I64, RTLIB::UREM_I128));
4878 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4879 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4884 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4885 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4895 RTLIB::MUL_I16, RTLIB::MUL_I32,
4896 RTLIB::MUL_I64, RTLIB::MUL_I128));
4899 switch (
Node->getSimpleValueType(0).SimpleTy) {
4903 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32,
Node,
false).first);
4906 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64,
Node,
false).first);
4909 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128,
Node,
false).first);
4916 SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
4919 DAG.makeStateFunctionCall(RTLIB::FESETENV,
Ptr, Chain, dl));
4926 DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl));
4933 DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl));
4939 EVT ModeVT =
Node->getValueType(0);
4941 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4942 SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr,
4943 Node->getOperand(0), dl);
4945 ModeVT, dl, Chain, StackPtr,
4955 EVT ModeVT =
Mode.getValueType();
4957 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4958 SDValue StInst = DAG.getStore(
4959 Node->getOperand(0), dl, Mode, StackPtr,
4962 DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl));
4970 EVT PtrTy = TLI.getPointerTy(
DL);
4972 Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
4973 Node->getOperand(0), dl));
4980 LLVM_DEBUG(
dbgs() <<
"Successfully converted node to libcall\n");
4989 MVT EltVT,
MVT NewEltVT) {
4991 MVT MidVT = OldEltsPerNewElt == 1
4998void SelectionDAGLegalize::PromoteNode(
SDNode *
Node) {
5001 MVT OVT =
Node->getSimpleValueType(0);
5007 OVT =
Node->getOperand(0).getSimpleValueType();
5014 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5015 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5016 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5017 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5018 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5019 OVT =
Node->getOperand(1).getSimpleValueType();
5022 OVT =
Node->getOperand(2).getSimpleValueType();
5023 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), OVT);
5025 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5026 switch (
Node->getOpcode()) {
5046 DAG.getConstant(TopBit, dl, NVT));
5050 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5064 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5067 DAG.getConstant(DiffBits, dl,
5068 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
5081 Results.push_back(PromoteLegalFP_TO_INT_SAT(
Node, dl));
5098 &&
"VAARG promotion is supported only for vectors or integer types");
5103 Tmp1 = DAG.getVAArg(NVT, dl, Chain,
Ptr,
Node->getOperand(2),
5104 Node->getConstantOperandVal(3));
5107 Tmp2 = DAG.
getNode(TruncOp, dl, OVT, Tmp1);
5111 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), Tmp2);
5112 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
5114 UpdatedNodes->insert(Tmp2.
getNode());
5115 UpdatedNodes->insert(Chain.
getNode());
5132 unsigned ExtOp, TruncOp;
5139 switch (
Node->getOpcode()) {
5155 if (TLI.isSExtCheaperThanZExt(OVT, NVT))
5164 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5165 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5167 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5168 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
5176 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5177 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5180 auto &
DL = DAG.getDataLayout();
5184 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(
DL, NVT)));
5190 unsigned ExtOp, TruncOp;
5191 if (
Node->getValueType(0).isVector() ||
5195 }
else if (
Node->getValueType(0).isInteger()) {
5202 Tmp1 =
Node->getOperand(0);
5204 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5205 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5207 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5210 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1);
5212 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1,
5213 DAG.getIntPtrConstant(0, dl));
5225 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5234 Node->getOperand(2));
5242 MVT CVT =
Node->getSimpleValueType(0);
5243 assert(CVT == OVT &&
"not handled");
5251 if (TLI.isCondCodeLegal(CCCode, CVT)) {
5252 Tmp1 =
Node->getOperand(0);
5253 Tmp2 =
Node->getOperand(1);
5255 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5256 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5259 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5260 Tmp4 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5270 DAG.getIntPtrConstant(0, dl,
true));
5282 TLI.isSExtCheaperThanZExt(
Node->getOperand(0).getValueType(), NVT))
5287 if (
Node->isStrictFPOpcode()) {
5289 std::tie(Tmp1, std::ignore) =
5290 DAG.getStrictFPExtendOrRound(
Node->getOperand(1), InChain, dl, NVT);
5291 std::tie(Tmp2, std::ignore) =
5292 DAG.getStrictFPExtendOrRound(
Node->getOperand(2), InChain, dl, NVT);
5294 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
5295 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
5296 Results.push_back(DAG.getNode(
Node->getOpcode(), dl, VTs,
5297 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5302 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5303 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5305 Tmp2,
Node->getOperand(2),
Node->getFlags()));
5312 cast<CondCodeSDNode>(
Node->getOperand(1))->get();
5315 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5316 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5318 Node->getOperand(0),
Node->getOperand(1),
5319 Tmp1, Tmp2,
Node->getOperand(4)));
5334 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
5338 DAG.getIntPtrConstant(0, dl,
true)));
5349 {
Node->getOperand(0),
Node->getOperand(1)});
5351 {
Node->getOperand(0),
Node->getOperand(2)});
5354 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5355 {Tmp3, Tmp1, Tmp2});
5357 {Tmp1.
getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5367 DAG.getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5368 DAG.getIntPtrConstant(0, dl,
true)));
5372 {
Node->getOperand(0),
Node->getOperand(1)});
5374 {
Node->getOperand(0),
Node->getOperand(2)});
5376 {
Node->getOperand(0),
Node->getOperand(3)});
5379 Tmp4 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5380 {Tmp4, Tmp1, Tmp2, Tmp3});
5382 {Tmp4.
getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5390 Tmp2 =
Node->getOperand(1);
5391 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5401 DAG.getIntPtrConstant(isTrunc, dl,
true)));
5406 {
Node->getOperand(0),
Node->getOperand(1)});
5407 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5410 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5420 DAG.getIntPtrConstant(0, dl,
true)));
5445 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5448 DAG.getIntPtrConstant(0, dl,
true)));
5466 {
Node->getOperand(0),
Node->getOperand(1)});
5467 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5470 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5485 "Invalid promote type for build_vector");
5491 for (
unsigned I = 0, E =
Node->getNumOperands();
I != E; ++
I) {
5520 "Invalid promote type for extract_vector_elt");
5527 EVT IdxVT =
Idx.getValueType();
5529 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
5535 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5536 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5544 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
5566 "Invalid promote type for insert_vector_elt");
5574 EVT IdxVT =
Idx.getValueType();
5577 SDValue Factor = DAG.getConstant(NewEltsPerOldElt,
SDLoc(), IdxVT);
5584 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5585 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5589 CastVal, IdxOffset);
5592 NewVec, Elt, InEltIdx);
5632 "unexpected promotion type");
5634 "unexpected atomic_swap with illegal type");
5658 "unexpected promotion type");
5660 "unexpected atomic_load with illegal type");
5671 MVT ScalarType =
Scalar.getSimpleValueType();
5675 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5680 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5683 DAG.getIntPtrConstant(0, dl,
true)));
5686 case ISD::VP_REDUCE_FADD:
5687 case ISD::VP_REDUCE_FMUL:
5688 case ISD::VP_REDUCE_FMAX:
5689 case ISD::VP_REDUCE_FMIN:
5690 case ISD::VP_REDUCE_SEQ_FADD:
5716 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes);
5723 bool AnyLegalized =
false;
5734 if (LegalizedNodes.
insert(
N).second) {
5735 AnyLegalized =
true;
5756 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes, &UpdatedNodes);
5763 return LegalizedNodes.
count(
N);
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
static MaybeAlign getAlign(Value *Ptr)
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI)
Return true if sincos libcall is available.
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
static MachineMemOperand * getStackAlignedMMO(SDValue StackPtr, MachineFunction &MF, bool isObjectScalable)
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
mir Rename Register Operands
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This is an SDNode representing atomic operations.
const SDValue & getBasePtr() const
const SDValue & getVal() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
const BasicBlock & back() const
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOStore
The memory access writes data.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetLowering & getTargetLoweringInfo() const
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
allnodes_const_iterator allnodes_end() const
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const DataLayout & getDataLayout() const
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
LLVMContext * getContext() const
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
Information about stack frame layout on the target.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
constexpr ScalarTy getFixedValue() const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ FRAME_TO_ARGS_OFFSET
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
@ RESET_FPENV
Set floating-point environment to default state.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ GLOBAL_OFFSET_TABLE
The address of the GOT.
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ SET_FPENV_MEM
Sets the current floating point environment.
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
@ Undef
Value of the register doesn't matter.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
int32_t ExponentType
A signed type to represent a floating point numbers unbiased exponent.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)