58#define DEBUG_TYPE "legalizedag"
64struct FloatSignAsInt {
87class SelectionDAGLegalize {
99 EVT getSetCCResultType(
EVT VT)
const {
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
166 void getSignAsIntValue(FloatSignAsInt &State,
const SDLoc &
DL,
168 SDValue modifySignAsInt(
const FloatSignAsInt &State,
const SDLoc &
DL,
215 dbgs() <<
" with: "; New->dump(&DAG));
218 "Replacing one node with another that produces a different number "
222 UpdatedNodes->
insert(New);
228 dbgs() <<
" with: "; New->dump(&DAG));
232 UpdatedNodes->
insert(New.getNode());
233 ReplacedNode(Old.getNode());
240 for (
unsigned i = 0, e = Old->
getNumValues(); i != e; ++i) {
244 UpdatedNodes->
insert(New[i].getNode());
251 dbgs() <<
" with: "; New->dump(&DAG));
255 UpdatedNodes->
insert(New.getNode());
256 ReplacedNode(Old.getNode());
266 bool isObjectScalable) {
268 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();
274 ObjectSize, MFI.getObjectAlign(FI));
281SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
286 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
288 assert(NumEltsGrowth &&
"Cannot promote to vector type with fewer elts!");
290 if (NumEltsGrowth == 1)
291 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
294 for (
unsigned i = 0; i != NumMaskElts; ++i) {
296 for (
unsigned j = 0;
j != NumEltsGrowth; ++
j) {
303 assert(NewMask.
size() == NumDestElts &&
"Non-integer NumEltsGrowth?");
304 assert(TLI.isShuffleMaskLegal(NewMask, NVT) &&
"Shuffle not legal?");
305 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
324 assert((VT == MVT::f64 || VT == MVT::f32) &&
"Invalid type expansion");
326 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
336 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
342 TLI.ShouldShrinkFPConstant(OrigVT)) {
345 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
353 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
354 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
363 OrigVT, dl, DAG.getEntryNode(), CPIdx,
371 EVT VT =
CP->getValueType(0);
372 SDValue CPIdx = DAG.getConstantPool(
CP->getConstantIntValue(),
373 TLI.getPointerTy(DAG.getDataLayout()));
374 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
376 VT, dl, DAG.getEntryNode(), CPIdx,
402 for (
unsigned i = 0; i != NumElts; ++i)
403 ShufOps.
push_back(i != InsertPos->getZExtValue() ? i : NumElts);
405 return DAG.getVectorShuffle(Vec.
getValueType(), dl, Vec, ScVec, ShufOps);
408 return ExpandInsertToVectorThroughStack(
Op);
435 TLI.isTypeLegal(MVT::i32)) {
437 bitcastToAPInt().zextOrTrunc(32),
438 SDLoc(CFP), MVT::i32);
439 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
440 ST->getOriginalAlign(), MMOFlags, AAInfo);
446 if (TLI.isTypeLegal(MVT::i64)) {
448 zextOrTrunc(64),
SDLoc(CFP), MVT::i64);
449 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
450 ST->getOriginalAlign(), MMOFlags, AAInfo);
453 if (TLI.isTypeLegal(MVT::i32) && !
ST->isVolatile()) {
460 if (DAG.getDataLayout().isBigEndian())
463 Lo = DAG.getStore(Chain, dl,
Lo,
Ptr,
ST->getPointerInfo(),
464 ST->getOriginalAlign(), MMOFlags, AAInfo);
466 Hi = DAG.getStore(Chain, dl,
Hi,
Ptr,
467 ST->getPointerInfo().getWithOffset(4),
468 ST->getOriginalAlign(), MMOFlags, AAInfo);
477void SelectionDAGLegalize::LegalizeStoreOps(
SDNode *
Node) {
486 if (!
ST->isTruncatingStore()) {
488 if (
SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
489 ReplaceNode(ST, OptStore);
494 MVT VT =
Value.getSimpleValueType();
495 switch (TLI.getOperationAction(
ISD::STORE, VT)) {
497 case TargetLowering::Legal: {
500 EVT MemVT =
ST->getMemoryVT();
502 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
503 *
ST->getMemOperand())) {
506 ReplaceNode(
SDValue(ST, 0), Result);
511 case TargetLowering::Custom: {
518 case TargetLowering::Promote: {
521 "Can only promote stores to same size type");
524 ST->getOriginalAlign(), MMOFlags, AAInfo);
534 EVT StVT =
ST->getMemoryVT();
537 auto &
DL = DAG.getDataLayout();
539 if (StWidth != StSize) {
544 Value = DAG.getZeroExtendInReg(
Value, dl, StVT);
546 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), NVT,
547 ST->getOriginalAlign(), MMOFlags, AAInfo);
553 unsigned LogStWidth =
Log2_32(StWidthBits);
555 unsigned RoundWidth = 1 << LogStWidth;
556 assert(RoundWidth < StWidthBits);
557 unsigned ExtraWidth = StWidthBits - RoundWidth;
558 assert(ExtraWidth < RoundWidth);
559 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
560 "Store size not an integral number of bytes!");
564 unsigned IncrementSize;
566 if (
DL.isLittleEndian()) {
569 Lo = DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(),
570 RoundVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
573 IncrementSize = RoundWidth / 8;
578 DAG.getConstant(RoundWidth, dl,
579 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
580 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
581 ST->getPointerInfo().getWithOffset(IncrementSize),
582 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
589 DAG.getConstant(ExtraWidth, dl,
590 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
591 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
ST->getPointerInfo(), RoundVT,
592 ST->getOriginalAlign(), MMOFlags, AAInfo);
595 IncrementSize = RoundWidth / 8;
597 DAG.getConstant(IncrementSize, dl,
598 Ptr.getValueType()));
600 ST->getPointerInfo().getWithOffset(IncrementSize),
601 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
608 switch (TLI.getTruncStoreAction(
ST->getValue().getValueType(), StVT)) {
610 case TargetLowering::Legal: {
611 EVT MemVT =
ST->getMemoryVT();
614 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
615 *
ST->getMemOperand())) {
617 ReplaceNode(
SDValue(ST, 0), Result);
621 case TargetLowering::Custom: {
627 case TargetLowering::Expand:
629 "Vector Stores are handled in LegalizeVectorOps");
634 if (TLI.isTypeLegal(StVT)) {
637 ST->getOriginalAlign(), MMOFlags, AAInfo);
642 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
645 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), StVT,
646 ST->getOriginalAlign(), MMOFlags, AAInfo);
655void SelectionDAGLegalize::LegalizeLoadOps(
SDNode *
Node) {
664 LLVM_DEBUG(
dbgs() <<
"Legalizing non-extending load operation\n");
665 MVT VT =
Node->getSimpleValueType(0);
669 switch (TLI.getOperationAction(
Node->getOpcode(), VT)) {
671 case TargetLowering::Legal: {
672 EVT MemVT =
LD->getMemoryVT();
676 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
677 *
LD->getMemOperand())) {
678 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
682 case TargetLowering::Custom:
683 if (
SDValue Res = TLI.LowerOperation(RVal, DAG)) {
689 case TargetLowering::Promote: {
690 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
692 "Can only promote loads to same size type");
694 SDValue Res = DAG.getLoad(NVT, dl, Chain,
Ptr,
LD->getMemOperand());
702 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), RVal);
703 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), RChain);
705 UpdatedNodes->insert(RVal.
getNode());
706 UpdatedNodes->insert(RChain.
getNode());
714 EVT SrcVT =
LD->getMemoryVT();
728 TLI.getLoadExtAction(ExtType,
Node->getValueType(0), MVT::i1) ==
729 TargetLowering::Promote)) {
743 Chain,
Ptr,
LD->getPointerInfo(), NVT,
744 LD->getOriginalAlign(), MMOFlags, AAInfo);
752 Result, DAG.getValueType(SrcVT));
756 Result.getValueType(), Result,
757 DAG.getValueType(SrcVT));
765 unsigned LogSrcWidth =
Log2_32(SrcWidthBits);
767 unsigned RoundWidth = 1 << LogSrcWidth;
768 assert(RoundWidth < SrcWidthBits);
769 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
770 assert(ExtraWidth < RoundWidth);
771 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
772 "Load size not an integral number of bytes!");
776 unsigned IncrementSize;
777 auto &
DL = DAG.getDataLayout();
779 if (
DL.isLittleEndian()) {
783 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
787 IncrementSize = RoundWidth / 8;
790 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
791 LD->getPointerInfo().getWithOffset(IncrementSize),
792 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
802 DAG.getConstant(RoundWidth, dl,
803 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
811 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
812 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
816 IncrementSize = RoundWidth / 8;
820 LD->getPointerInfo().getWithOffset(IncrementSize),
821 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
831 DAG.getConstant(ExtraWidth, dl,
832 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
840 bool isCustom =
false;
841 switch (TLI.getLoadExtAction(ExtType,
Node->getValueType(0),
844 case TargetLowering::Custom:
847 case TargetLowering::Legal:
859 EVT MemVT =
LD->getMemoryVT();
861 if (!TLI.allowsMemoryAccess(*DAG.getContext(),
DL, MemVT,
862 *
LD->getMemOperand())) {
863 std::tie(
Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
868 case TargetLowering::Expand: {
869 EVT DestVT =
Node->getValueType(0);
875 (TLI.isTypeLegal(SrcVT) ||
876 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
883 SrcVT,
LD->getMemOperand());
886 Value = DAG.getNode(ExtendOp, dl,
Node->getValueType(0), Load);
887 Chain =
Load.getValue(1);
896 if (SVT == MVT::f16 || SVT == MVT::bf16) {
902 Ptr, ISrcVT,
LD->getMemOperand());
906 Chain =
Result.getValue(1);
912 "Vector Loads are handled in LegalizeVectorOps");
919 "EXTLOAD should always be supported!");
923 Node->getValueType(0),
925 LD->getMemOperand());
930 Result, DAG.getValueType(SrcVT));
932 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
934 Chain =
Result.getValue(1);
945 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
947 UpdatedNodes->insert(
Value.getNode());
948 UpdatedNodes->insert(Chain.
getNode());
955void SelectionDAGLegalize::LegalizeOp(
SDNode *
Node) {
964 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
965 assert(TLI.getTypeAction(*DAG.getContext(),
Node->getValueType(i)) ==
966 TargetLowering::TypeLegal &&
967 "Unexpected illegal type!");
970 assert((TLI.getTypeAction(*DAG.getContext(),
Op.getValueType()) ==
971 TargetLowering::TypeLegal ||
974 "Unexpected illegal type!");
979 bool SimpleFinishLegalizing =
true;
980 switch (
Node->getOpcode()) {
985 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
988 Action = TLI.getOperationAction(
Node->getOpcode(),
989 Node->getValueType(0));
992 Action = TLI.getOperationAction(
Node->getOpcode(),
993 Node->getValueType(0));
994 if (Action != TargetLowering::Promote)
995 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
999 Action = TLI.getOperationAction(
Node->getOpcode(),
1000 Node->getOperand(1).getValueType());
1011 Action = TLI.getOperationAction(
Node->getOpcode(),
1012 Node->getOperand(0).getValueType());
1025 Action = TLI.getOperationAction(
Node->getOpcode(),
1026 Node->getOperand(1).getValueType());
1029 EVT InnerType = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1030 Action = TLI.getOperationAction(
Node->getOpcode(), InnerType);
1034 Action = TLI.getOperationAction(
Node->getOpcode(),
1035 Node->getOperand(1).getValueType());
1044 unsigned Opc =
Node->getOpcode();
1049 : (Opc ==
ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1051 unsigned CompareOperand = Opc ==
ISD::BR_CC ? 2
1055 MVT OpVT =
Node->getOperand(CompareOperand).getSimpleValueType();
1057 cast<CondCodeSDNode>(
Node->getOperand(CCOperand))->get();
1058 Action = TLI.getCondCodeAction(CCCode, OpVT);
1059 if (Action == TargetLowering::Legal) {
1061 Action = TLI.getOperationAction(
Node->getOpcode(),
1062 Node->getValueType(0));
1064 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
1072 SimpleFinishLegalizing =
false;
1079 SimpleFinishLegalizing =
false;
1092 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1093 if (Action == TargetLowering::Legal)
1094 Action = TargetLowering::Expand;
1104 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1105 if (Action == TargetLowering::Legal)
1106 Action = TargetLowering::Custom;
1111 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1117 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::i64);
1124 Action = TargetLowering::Legal;
1127 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1128 if (Action == TargetLowering::Expand) {
1132 Node->getOperand(0));
1139 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1140 if (Action == TargetLowering::Expand) {
1144 Node->getOperand(0));
1158 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1168 unsigned Scale =
Node->getConstantOperandVal(2);
1169 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
1170 Node->getValueType(0), Scale);
1174 Action = TLI.getOperationAction(
Node->getOpcode(),
1175 cast<MaskedScatterSDNode>(
Node)->getValue().getValueType());
1178 Action = TLI.getOperationAction(
Node->getOpcode(),
1179 cast<MaskedStoreSDNode>(
Node)->getValue().getValueType());
1181 case ISD::VP_SCATTER:
1182 Action = TLI.getOperationAction(
1184 cast<VPScatterSDNode>(
Node)->getValue().getValueType());
1187 Action = TLI.getOperationAction(
1189 cast<VPStoreSDNode>(
Node)->getValue().getValueType());
1191 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1192 Action = TLI.getOperationAction(
1194 cast<VPStridedStoreSDNode>(
Node)->getValue().getValueType());
1212 Action = TLI.getOperationAction(
1213 Node->getOpcode(),
Node->getOperand(0).getValueType());
1217 case ISD::VP_REDUCE_FADD:
1218 case ISD::VP_REDUCE_FMUL:
1219 case ISD::VP_REDUCE_ADD:
1220 case ISD::VP_REDUCE_MUL:
1221 case ISD::VP_REDUCE_AND:
1222 case ISD::VP_REDUCE_OR:
1223 case ISD::VP_REDUCE_XOR:
1224 case ISD::VP_REDUCE_SMAX:
1225 case ISD::VP_REDUCE_SMIN:
1226 case ISD::VP_REDUCE_UMAX:
1227 case ISD::VP_REDUCE_UMIN:
1228 case ISD::VP_REDUCE_FMAX:
1229 case ISD::VP_REDUCE_FMIN:
1230 case ISD::VP_REDUCE_FMAXIMUM:
1231 case ISD::VP_REDUCE_FMINIMUM:
1232 case ISD::VP_REDUCE_SEQ_FADD:
1233 case ISD::VP_REDUCE_SEQ_FMUL:
1234 Action = TLI.getOperationAction(
1235 Node->getOpcode(),
Node->getOperand(1).getValueType());
1237 case ISD::VP_CTTZ_ELTS:
1238 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
1239 Action = TLI.getOperationAction(
Node->getOpcode(),
1240 Node->getOperand(0).getValueType());
1244 Action = TLI.getCustomOperationAction(*
Node);
1246 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1251 if (SimpleFinishLegalizing) {
1253 switch (
Node->getOpcode()) {
1272 NewNode = DAG.UpdateNodeOperands(
Node, Op0, SAO);
1292 NewNode = DAG.UpdateNodeOperands(
Node, Op0, Op1, SAO);
1298 if (NewNode !=
Node) {
1299 ReplaceNode(
Node, NewNode);
1303 case TargetLowering::Legal:
1306 case TargetLowering::Custom:
1314 if (
Node->getNumValues() == 1) {
1318 Node->getValueType(0) == MVT::Glue) &&
1319 "Type mismatch for custom legalized operation");
1327 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i) {
1331 Node->getValueType(i) == MVT::Glue) &&
1332 "Type mismatch for custom legalized operation");
1336 ReplaceNode(
Node, ResultVals.
data());
1341 case TargetLowering::Expand:
1342 if (ExpandNode(
Node))
1345 case TargetLowering::LibCall:
1346 ConvertNodeToLibcall(
Node);
1348 case TargetLowering::Promote:
1354 switch (
Node->getOpcode()) {
1367 return LegalizeLoadOps(
Node);
1369 return LegalizeStoreOps(
Node);
1373SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(
SDValue Op) {
1393 if (
ST->isIndexed() ||
ST->isTruncatingStore() ||
1394 ST->getValue() != Vec)
1399 if (!
ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1408 ST->hasPredecessor(
Op.getNode()))
1421 StackPtr = DAG.CreateStackTemporary(VecVT);
1424 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO);
1428 Align ElementAlignment =
1429 std::min(cast<StoreSDNode>(Ch)->
getAlign(),
1430 DAG.getDataLayout().getPrefTypeAlign(
1431 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1433 if (
Op.getValueType().isVector()) {
1434 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1435 Op.getValueType(),
Idx);
1436 NewLoad = DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr,
1439 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1440 NewLoad = DAG.getExtLoad(
ISD::EXTLOAD, dl,
Op.getValueType(), Ch, StackPtr,
1446 DAG.ReplaceAllUsesOfValueWith(Ch,
SDValue(NewLoad.
getNode(), 1));
1452 NewLoadOperands[0] = Ch;
1454 SDValue(DAG.UpdateNodeOperands(NewLoad.
getNode(), NewLoadOperands), 0);
1458SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(
SDValue Op) {
1459 assert(
Op.getValueType().isVector() &&
"Non-vector insert subvector!");
1470 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1475 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1478 Idx = DAG.getFreeze(
Idx);
1483 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, PartVT,
Idx);
1487 Ch, dl, Part, SubStackPtr,
1491 TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1494 Ch = DAG.getTruncStore(
1495 Ch, dl, Part, SubStackPtr,
1501 return DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1507 "Unexpected opcode!");
1513 EVT VT =
Node->getValueType(0);
1515 :
Node->getOperand(0).getValueType();
1517 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1518 int FI = cast<FrameIndexSDNode>(FIPtr.
getNode())->getIndex();
1525 assert(TypeByteSize > 0 &&
"Vector element type too small for stack store!");
1529 bool Truncate = isa<BuildVectorSDNode>(
Node) &&
1530 MemVT.
bitsLT(
Node->getOperand(0).getValueType());
1533 for (
unsigned i = 0, e =
Node->getNumOperands(); i != e; ++i) {
1535 if (
Node->getOperand(i).isUndef())
continue;
1537 unsigned Offset = TypeByteSize*i;
1543 Stores.
push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1547 Stores.
push_back(DAG.getStore(DAG.getEntryNode(), dl,
Node->getOperand(i),
1552 if (!Stores.
empty())
1555 StoreChain = DAG.getEntryNode();
1558 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1564void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1567 EVT FloatVT =
Value.getValueType();
1569 State.FloatVT = FloatVT;
1572 if (TLI.isTypeLegal(IVT)) {
1575 State.SignBit = NumBits - 1;
1581 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1584 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1589 State.Chain = DAG.getStore(DAG.getEntryNode(),
DL,
Value, State.FloatPtr,
1590 State.FloatPointerInfo);
1597 State.IntPointerInfo = State.FloatPointerInfo;
1600 unsigned ByteOffset = (NumBits / 8) - 1;
1607 State.IntPtr = IntPtr;
1608 State.IntValue = DAG.getExtLoad(
ISD::EXTLOAD,
DL, LoadTy, State.Chain, IntPtr,
1609 State.IntPointerInfo, MVT::i8);
1616SDValue SelectionDAGLegalize::modifySignAsInt(
const FloatSignAsInt &State,
1623 SDValue Chain = DAG.getTruncStore(State.Chain,
DL, NewIntValue, State.IntPtr,
1624 State.IntPointerInfo, MVT::i8);
1625 return DAG.getLoad(State.FloatVT,
DL, Chain, State.FloatPtr,
1626 State.FloatPointerInfo);
1635 FloatSignAsInt SignAsInt;
1636 getSignAsIntValue(SignAsInt,
DL, Sign);
1638 EVT IntVT = SignAsInt.IntValue.getValueType();
1639 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1645 if (TLI.isOperationLegalOrCustom(
ISD::FABS, FloatVT) &&
1646 TLI.isOperationLegalOrCustom(
ISD::FNEG, FloatVT)) {
1649 SDValue Cond = DAG.getSetCC(
DL, getSetCCResultType(IntVT), SignBit,
1651 return DAG.getSelect(
DL, FloatVT,
Cond, NegValue, AbsValue);
1655 FloatSignAsInt MagAsInt;
1656 getSignAsIntValue(MagAsInt,
DL, Mag);
1657 EVT MagVT = MagAsInt.IntValue.getValueType();
1658 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask,
DL, MagVT);
1663 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1664 EVT ShiftVT = IntVT;
1670 if (ShiftAmount > 0) {
1671 SDValue ShiftCnst = DAG.getConstant(ShiftAmount,
DL, ShiftVT);
1673 }
else if (ShiftAmount < 0) {
1674 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount,
DL, ShiftVT);
1684 return modifySignAsInt(MagAsInt,
DL, CopiedSign);
1690 FloatSignAsInt SignAsInt;
1691 getSignAsIntValue(SignAsInt,
DL,
Node->getOperand(0));
1692 EVT IntVT = SignAsInt.IntValue.getValueType();
1695 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1700 return modifySignAsInt(SignAsInt,
DL, SignFlip);
1708 EVT FloatVT =
Value.getValueType();
1715 FloatSignAsInt ValueAsInt;
1716 getSignAsIntValue(ValueAsInt,
DL,
Value);
1717 EVT IntVT = ValueAsInt.IntValue.getValueType();
1718 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask,
DL, IntVT);
1721 return modifySignAsInt(ValueAsInt,
DL, ClearedSign);
1724void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(
SDNode*
Node,
1726 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1727 assert(SPReg &&
"Target cannot require DYNAMIC_STACKALLOC expansion and"
1728 " not tell us which reg is the stack pointer!");
1730 EVT VT =
Node->getValueType(0);
1738 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1741 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1743 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1751 if (Alignment > StackAlign)
1753 DAG.getConstant(-Alignment.
value(), dl, VT));
1754 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);
1756 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0,
SDValue(), dl);
1768 return EmitStackConvert(
SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1776 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1779 if ((SrcVT.
bitsGT(SlotVT) &&
1780 !TLI.isTruncStoreLegalOrCustom(
SrcOp.getValueType(), SlotVT)) ||
1781 (SlotVT.
bitsLT(DestVT) &&
1782 !TLI.isLoadExtLegalOrCustom(
ISD::EXTLOAD, DestVT, SlotVT)))
1786 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1787 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1799 if (SrcVT.
bitsGT(SlotVT))
1800 Store = DAG.getTruncStore(Chain, dl,
SrcOp, FIPtr, PtrInfo,
1804 Store = DAG.getStore(Chain, dl,
SrcOp, FIPtr, PtrInfo, SrcAlign);
1808 if (SlotVT.
bitsEq(DestVT))
1809 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1812 return DAG.getExtLoad(
ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1825 SDValue Ch = DAG.getTruncStore(
1826 DAG.getEntryNode(), dl,
Node->getOperand(0), StackPtr,
1828 Node->getValueType(0).getVectorElementType());
1830 Node->getValueType(0), dl, Ch, StackPtr,
1837 unsigned NumElems =
Node->getNumOperands();
1839 EVT VT =
Node->getValueType(0);
1851 for (
unsigned i = 0; i < NumElems; ++i) {
1862 while (IntermedVals.
size() > 2) {
1863 NewIntermedVals.
clear();
1864 for (
unsigned i = 0, e = (IntermedVals.
size() & ~1u); i < e; i += 2) {
1870 FinalIndices.
reserve(IntermedVals[i].second.
size() +
1871 IntermedVals[i+1].second.
size());
1874 for (
unsigned j = 0, f = IntermedVals[i].second.
size(); j != f;
1877 FinalIndices.
push_back(IntermedVals[i].second[j]);
1879 for (
unsigned j = 0, f = IntermedVals[i+1].second.
size(); j != f;
1881 ShuffleVec[k] = NumElems + j;
1882 FinalIndices.
push_back(IntermedVals[i+1].second[j]);
1888 IntermedVals[i+1].first,
1893 std::make_pair(Shuffle, std::move(FinalIndices)));
1898 if ((IntermedVals.
size() & 1) != 0)
1901 IntermedVals.
swap(NewIntermedVals);
1905 "Invalid number of intermediate vectors");
1906 SDValue Vec1 = IntermedVals[0].first;
1908 if (IntermedVals.
size() > 1)
1909 Vec2 = IntermedVals[1].first;
1914 for (
unsigned i = 0, e = IntermedVals[0].second.
size(); i != e; ++i)
1915 ShuffleVec[IntermedVals[0].second[i]] = i;
1916 for (
unsigned i = 0, e = IntermedVals[1].second.
size(); i != e; ++i)
1917 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1931 unsigned NumElems =
Node->getNumOperands();
1934 EVT VT =
Node->getValueType(0);
1935 EVT OpVT =
Node->getOperand(0).getValueType();
1940 bool isOnlyLowElement =
true;
1941 bool MoreThanTwoValues =
false;
1943 for (
unsigned i = 0; i < NumElems; ++i) {
1948 isOnlyLowElement =
false;
1949 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1954 }
else if (!Value2.
getNode()) {
1957 }
else if (V != Value1 && V != Value2) {
1958 MoreThanTwoValues =
true;
1963 return DAG.getUNDEF(VT);
1965 if (isOnlyLowElement)
1971 for (
unsigned i = 0, e = NumElems; i !=
e; ++i) {
1973 dyn_cast<ConstantFPSDNode>(
Node->getOperand(i))) {
1976 dyn_cast<ConstantSDNode>(
Node->getOperand(i))) {
1995 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1996 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1998 VT, dl, DAG.getEntryNode(), CPIdx,
2004 for (
unsigned i = 0; i < NumElems; ++i) {
2005 if (
Node->getOperand(i).isUndef())
2010 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.
size())) {
2011 if (!MoreThanTwoValues) {
2013 for (
unsigned i = 0; i < NumElems; ++i) {
2017 ShuffleVec[i] =
V == Value1 ? 0 : NumElems;
2019 if (TLI.isShuffleMaskLegal(ShuffleVec,
Node->getValueType(0))) {
2026 Vec2 = DAG.getUNDEF(VT);
2029 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2039 return ExpandVectorBuildThroughStack(
Node);
2044 EVT VT =
Node->getValueType(0);
2047 return DAG.getSplatBuildVector(VT,
DL, SplatVal);
2058 EVT CodePtrTy = TLI.getPointerTy(DAG.getDataLayout());
2060 if (
const char *LibcallName = TLI.getLibcallName(LC))
2061 Callee = DAG.getExternalSymbol(LibcallName, CodePtrTy);
2063 Callee = DAG.getUNDEF(CodePtrTy);
2064 DAG.getContext()->emitError(
Twine(
"no libcall available for ") +
2065 Node->getOperationName(&DAG));
2068 EVT RetVT =
Node->getValueType(0);
2075 SDValue InChain = DAG.getEntryNode();
2080 const Function &
F = DAG.getMachineFunction().getFunction();
2082 TLI.isInTailCallPosition(DAG,
Node, TCChain) &&
2083 (
RetTy ==
F.getReturnType() ||
F.getReturnType()->isVoidTy());
2088 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT,
isSigned);
2091 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2093 .setTailCall(isTailCall)
2094 .setSExtResult(signExtend)
2095 .setZExtResult(!signExtend)
2096 .setIsPostTypeLegalization(
true);
2098 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2103 return {DAG.getRoot(), DAG.getRoot()};
2115 EVT ArgVT =
Op.getValueType();
2119 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT,
isSigned);
2120 Entry.IsZExt = !Entry.IsSExt;
2121 Args.push_back(Entry);
2124 return ExpandLibCall(LC,
Node, std::move(Args),
isSigned);
2127void SelectionDAGLegalize::ExpandFrexpLibCall(
2130 EVT VT =
Node->getValueType(0);
2131 EVT ExpVT =
Node->getValueType(1);
2139 FPArgEntry.
Node = FPOp;
2140 FPArgEntry.
Ty = ArgTy;
2142 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2144 PtrArgEntry.
Node = StackSlot;
2145 PtrArgEntry.
Ty = PointerType::get(*DAG.getContext(),
2146 DAG.getDataLayout().getAllocaAddrSpace());
2151 auto [
Call, Chain] = ExpandLibCall(LC,
Node, std::move(Args),
false);
2155 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2159 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2161 LoadExp.
getValue(1), DAG.getRoot());
2162 DAG.setRoot(OutputChain);
2168void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2171 if (LC == RTLIB::UNKNOWN_LIBCALL)
2174 if (
Node->isStrictFPOpcode()) {
2175 EVT RetVT =
Node->getValueType(0);
2179 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2182 Node->getOperand(0));
2184 Results.push_back(Tmp.second);
2186 SDValue Tmp = ExpandLibCall(LC,
Node,
false).first;
2192void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2200 Call_F32, Call_F64, Call_F80,
2201 Call_F128, Call_PPCF128);
2212 switch (
Node->getSimpleValueType(0).SimpleTy) {
2214 case MVT::i8: LC = Call_I8;
break;
2215 case MVT::i16: LC = Call_I16;
break;
2216 case MVT::i32: LC = Call_I32;
break;
2217 case MVT::i64: LC = Call_I64;
break;
2218 case MVT::i128: LC = Call_I128;
break;
2225void SelectionDAGLegalize::ExpandArgFPLibCall(
SDNode*
Node,
2232 EVT InVT =
Node->getOperand(
Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2234 Call_F32, Call_F64, Call_F80,
2235 Call_F128, Call_PPCF128);
2241SelectionDAGLegalize::ExpandDivRemLibCall(
SDNode *
Node,
2243 unsigned Opcode =
Node->getOpcode();
2247 switch (
Node->getSimpleValueType(0).SimpleTy) {
2249 case MVT::i8: LC=
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
2250 case MVT::i16: LC=
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
2251 case MVT::i32: LC=
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
2252 case MVT::i64: LC=
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
2253 case MVT::i128: LC=
isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
2259 SDValue InChain = DAG.getEntryNode();
2261 EVT RetVT =
Node->getValueType(0);
2267 EVT ArgVT =
Op.getValueType();
2273 Args.push_back(Entry);
2277 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2279 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2282 Args.push_back(Entry);
2284 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2285 TLI.getPointerTy(DAG.getDataLayout()));
2291 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2296 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2308 switch (
Node->getSimpleValueType(0).SimpleTy) {
2310 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2311 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2312 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2313 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2314 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2337SelectionDAGLegalize::ExpandSinCosLibCall(
SDNode *
Node,
2340 switch (
Node->getSimpleValueType(0).SimpleTy) {
2342 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2343 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2344 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2345 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2346 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2352 SDValue InChain = DAG.getEntryNode();
2354 EVT RetVT =
Node->getValueType(0);
2361 Entry.Node =
Node->getOperand(0);
2363 Entry.IsSExt =
false;
2364 Entry.IsZExt =
false;
2365 Args.push_back(Entry);
2368 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2369 Entry.Node = SinPtr;
2370 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2371 Entry.IsSExt =
false;
2372 Entry.IsZExt =
false;
2373 Args.push_back(Entry);
2376 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2377 Entry.Node = CosPtr;
2378 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2379 Entry.IsSExt =
false;
2380 Entry.IsZExt =
false;
2381 Args.push_back(Entry);
2383 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2384 TLI.getPointerTy(DAG.getDataLayout()));
2388 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2389 TLI.getLibcallCallingConv(LC),
Type::getVoidTy(*DAG.getContext()), Callee,
2392 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2402 EVT VT =
Node->getValueType(0);
2405 EVT ExpVT =
N.getValueType();
2407 if (AsIntVT ==
EVT())
2420 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2425 const int Precision = APFloat::semanticsPrecision(FltSem);
2427 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2428 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2430 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2432 const APFloat One(FltSem,
"1.0");
2433 APFloat ScaleUpK =
scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2437 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2446 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2452 DAG.getSetCC(dl, SetCCVT,
N, DoubleMaxExp,
ISD::SETUGT);
2454 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2466 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2467 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2472 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2477 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2481 SDValue ScaleDownTwice = DAG.getSetCC(
2482 dl, SetCCVT,
N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2494 DAG.getNode(
ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small,
X));
2498 DAG.getNode(
ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small,
N));
2503 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2504 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2507 ExponentShiftAmt, NUW_NSW);
2509 return DAG.getNode(
ISD::FMUL, dl, VT, NewX, AsFP);
2516 EVT ExpVT =
Node->getValueType(1);
2518 if (AsIntVT ==
EVT())
2523 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2540 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2544 SDValue SmallestNormalizedInt = DAG.getConstant(
2550 DAG.getConstant(
APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2556 FractSignMaskVal.
setBit(BitSize - 1);
2559 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2561 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2563 const APFloat One(FltSem,
"1.0");
2567 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2569 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2573 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2579 SDValue AddNegSmallestNormal =
2581 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2585 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt,
ISD::SETULT);
2587 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2602 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2605 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2608 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2615 SDValue FPHalf = DAG.getConstant(
Half.bitcastToAPInt(), dl, AsIntVT);
2628 return DAG.getMergeValues({Result0, Result1}, dl);
2639 EVT DestVT =
Node->getValueType(0);
2641 unsigned OpNo =
Node->isStrictFPOpcode() ? 1 : 0;
2647 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2648 (DestVT.
bitsLE(MVT::f64) ||
2652 LLVM_DEBUG(
dbgs() <<
"32-bit [signed|unsigned] integer to float/double "
2656 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2663 DAG.getConstant(0x80000000u, dl, MVT::i32));
2666 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2669 if (DAG.getDataLayout().isBigEndian())
2672 SDValue MemChain = DAG.getEntryNode();
2675 SDValue Store1 = DAG.getStore(MemChain, dl,
Lo, StackSlot,
2688 SDValue Bias = DAG.getConstantFP(
2689 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2690 : llvm::bit_cast<double>(0x4330000000000000ULL),
2695 if (
Node->isStrictFPOpcode()) {
2697 {
Node->getOperand(0), Load, Bias});
2700 std::pair<SDValue, SDValue> ResultPair;
2702 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2703 Result = ResultPair.first;
2704 Chain = ResultPair.second;
2710 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2719 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2720 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2721 LLVM_DEBUG(
dbgs() <<
"Converting unsigned i32/i64 to f32/f64\n");
2736 EVT SetCCVT = getSetCCResultType(SrcVT);
2738 SDValue SignBitTest = DAG.getSetCC(
2739 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT),
ISD::SETLT);
2741 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2742 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2744 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2749 if (
Node->isStrictFPOpcode()) {
2752 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest,
Or, Op0);
2754 {
Node->getOperand(0), InCvt });
2762 Flags.setNoFPExcept(
Node->getFlags().hasNoFPExcept());
2763 Fast->setFlags(Flags);
2764 Flags.setNoFPExcept(
true);
2772 return DAG.getSelect(dl, DestVT, SignBitTest, Slow,
Fast);
2776 if (!TLI.isOperationLegalOrCustom(
2783 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2785 "Cannot perform lossless SINT_TO_FP!");
2788 if (
Node->isStrictFPOpcode()) {
2790 {
Node->getOperand(0), Op0 });
2794 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2797 Four = DAG.getIntPtrConstant(4, dl);
2798 SDValue CstOffset = DAG.getSelect(dl,
Zero.getValueType(),
2799 SignSet, Four, Zero);
2808 case MVT::i8 : FF = 0x43800000ULL;
break;
2809 case MVT::i16: FF = 0x47800000ULL;
break;
2810 case MVT::i32: FF = 0x4F800000ULL;
break;
2811 case MVT::i64: FF = 0x5F800000ULL;
break;
2813 if (DAG.getDataLayout().isLittleEndian())
2815 Constant *FudgeFactor = ConstantInt::get(
2819 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2820 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2824 if (DestVT == MVT::f32)
2825 FudgeInReg = DAG.getLoad(
2826 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2835 LegalizeOp(
Load.getNode());
2839 if (
Node->isStrictFPOpcode()) {
2841 { Tmp1.
getValue(1), Tmp1, FudgeInReg });
2842 Chain =
Result.getValue(1);
2846 return DAG.getNode(
ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2854void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2856 bool IsStrict =
N->isStrictFPOpcode();
2859 EVT DestVT =
N->getValueType(0);
2860 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2867 unsigned OpToUse = 0;
2875 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2883 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2895 DAG.
getNode(OpToUse, dl, {DestVT, MVT::Other},
2898 dl, NewInTy, LegalOp)});
2905 DAG.getNode(OpToUse, dl, DestVT,
2907 dl, NewInTy, LegalOp)));
2915void SelectionDAGLegalize::PromoteLegalFP_TO_INT(
SDNode *
N,
const SDLoc &dl,
2917 bool IsStrict =
N->isStrictFPOpcode();
2920 EVT DestVT =
N->getValueType(0);
2921 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2923 EVT NewOutTy = DestVT;
2925 unsigned OpToUse = 0;
2935 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2940 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2949 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2950 Operation = DAG.getNode(OpToUse, dl, VTs,
N->getOperand(0), LegalOp);
2952 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2967 unsigned Opcode =
Node->getOpcode();
2970 EVT NewOutTy =
Node->getValueType(0);
2975 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2982 Node->getOperand(1));
2988 EVT VT =
Op.getValueType();
2989 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2994 if (TLI.isOperationLegalOrPromote(
ISD::CTPOP, VT)) {
3000 DAG.getConstant(1ULL << (--i), dl, ShVT));
3005 return DAG.getNode(
ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
3009 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
3010 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
3011 MVT ScalarVT =
Node->getSimpleValueType(0);
3019 assert(
Node->getOperand(0).getValueType().isFloatingPoint() &&
3020 "Only FP promotion is supported");
3024 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
3025 if (
Node->getOperand(j).getValueType().isVector() &&
3030 assert(
Node->getOperand(j).getValueType().isFloatingPoint() &&
3031 "Only FP promotion is supported");
3043 DAG.getIntPtrConstant(0,
DL,
true));
3046bool SelectionDAGLegalize::ExpandNode(
SDNode *
Node) {
3050 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3052 switch (
Node->getOpcode()) {
3054 if ((Tmp1 = TLI.expandABS(
Node, DAG)))
3059 if ((Tmp1 = TLI.expandABD(
Node, DAG)))
3063 if ((Tmp1 = TLI.expandCTPOP(
Node, DAG)))
3068 if ((Tmp1 = TLI.expandCTLZ(
Node, DAG)))
3073 if ((Tmp1 = TLI.expandCTTZ(
Node, DAG)))
3077 if ((Tmp1 = TLI.expandBITREVERSE(
Node, DAG)))
3081 if ((Tmp1 = TLI.expandBSWAP(
Node, DAG)))
3085 Results.push_back(ExpandPARITY(
Node->getOperand(0), dl));
3090 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3093 SDValue CfaArg = DAG.getSExtOrTrunc(
Node->getOperand(0), dl,
3094 TLI.getPointerTy(DAG.getDataLayout()));
3102 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3108 Results.push_back(DAG.getConstant(1, dl,
Node->getValueType(0)));
3125 DAG.getConstant(0, dl,
Node->getValueType(0)));
3131 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3137 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3138 SDValue Swap = DAG.getAtomicCmpSwap(
3140 Node->getOperand(0),
Node->getOperand(1), Zero, Zero,
3141 cast<AtomicSDNode>(
Node)->getMemOperand());
3150 Node->getOperand(0),
Node->getOperand(2),
Node->getOperand(1),
3151 cast<AtomicSDNode>(
Node)->getMemOperand());
3159 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3160 SDValue Res = DAG.getAtomicCmpSwap(
3162 Node->getOperand(0),
Node->getOperand(1),
Node->getOperand(2),
3163 Node->getOperand(3), cast<MemSDNode>(
Node)->getMemOperand());
3169 EVT AtomicType = cast<AtomicSDNode>(
Node)->getMemoryVT();
3170 EVT OuterType =
Node->getValueType(0);
3171 switch (TLI.getExtendForAtomicOps()) {
3174 DAG.getValueType(AtomicType));
3176 Node->getOperand(2), DAG.getValueType(AtomicType));
3181 DAG.getValueType(AtomicType));
3182 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3186 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3187 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3203 EVT VT =
Node->getValueType(0);
3207 cast<VTSDNode>(
RHS->getOperand(1))->getVT() == AN->
getMemoryVT())
3208 RHS =
RHS->getOperand(0);
3212 Node->getOperand(0),
Node->getOperand(1),
3222 for (
unsigned i = 0; i <
Node->getNumValues(); i++)
3226 EVT VT =
Node->getValueType(0);
3228 Results.push_back(DAG.getConstant(0, dl, VT));
3231 Results.push_back(DAG.getConstantFP(0, dl, VT));
3238 if (TLI.isStrictFPEnabled())
3242 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3243 Node->getValueType(0))
3244 == TargetLowering::Legal)
3248 if ((Tmp1 = EmitStackConvert(
Node->getOperand(1),
Node->getValueType(0),
3249 Node->getValueType(0), dl,
3250 Node->getOperand(0)))) {
3252 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_ROUND node\n");
3257 if ((Tmp1 = TLI.expandFP_ROUND(
Node, DAG))) {
3265 if ((Tmp1 = EmitStackConvert(
Node->getOperand(0),
Node->getValueType(0),
3266 Node->getValueType(0), dl)))
3272 if (TLI.isStrictFPEnabled())
3276 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3277 Node->getValueType(0))
3278 == TargetLowering::Legal)
3282 if ((Tmp1 = EmitStackConvert(
3283 Node->getOperand(1),
Node->getOperand(1).getValueType(),
3284 Node->getValueType(0), dl,
Node->getOperand(0)))) {
3286 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_EXTEND node\n");
3292 EVT SrcVT =
Op.getValueType();
3293 EVT DstVT =
Node->getValueType(0);
3299 if ((Tmp1 = EmitStackConvert(
Op, SrcVT, DstVT, dl)))
3309 if (
Op.getValueType() == MVT::bf16) {
3313 Op = DAG.getAnyExtOrTrunc(
Op, dl, MVT::i32);
3317 DAG.getConstant(16, dl,
3318 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3321 if (
Node->getValueType(0) != MVT::f32)
3328 if (
Op.getValueType() != MVT::f32)
3330 DAG.getIntPtrConstant(0, dl,
true));
3332 if (!DAG.isKnownNeverSNaN(
Op)) {
3337 DAG.getConstant(16, dl,
3338 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3341 if (
Node->getValueType(0) == MVT::bf16) {
3345 Op = DAG.getAnyExtOrTrunc(
Op, dl,
Node->getValueType(0));
3351 EVT ExtraVT = cast<VTSDNode>(
Node->getOperand(1))->getVT();
3352 EVT VT =
Node->getValueType(0);
3362 SDValue One = DAG.getConstant(1, dl, VT);
3372 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3375 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3377 Node->getOperand(0), ShiftCst);
3384 if (TLI.expandUINT_TO_FP(
Node, Tmp1, Tmp2, DAG)) {
3386 if (
Node->isStrictFPOpcode())
3393 if ((Tmp1 = ExpandLegalINT_TO_FP(
Node, Tmp2))) {
3395 if (
Node->isStrictFPOpcode())
3400 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG))
3404 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG)) {
3406 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_SINT node\n");
3411 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG))
3415 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG)) {
3417 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node,1), Tmp2);
3420 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_UINT node\n");
3426 Results.push_back(TLI.expandFP_TO_INT_SAT(
Node, DAG));
3436 if (
Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3439 Node->getOperand(0));
3441 Tmp1 = ExpandExtractFromVectorThroughStack(
SDValue(
Node, 0));
3451 Results.push_back(ExpandVectorBuildThroughStack(
Node));
3463 EVT VT =
Node->getValueType(0);
3467 if (!TLI.isTypeLegal(EltVT)) {
3468 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3473 if (NewEltVT.
bitsLT(EltVT)) {
3489 unsigned int factor =
3497 for (
unsigned fi = 0; fi < factor; ++fi)
3501 for (
unsigned fi = 0; fi < factor; ++fi)
3512 for (
unsigned i = 0; i != NumElems; ++i) {
3520 DAG.getVectorIdxConstant(
Idx, dl)));
3524 DAG.getVectorIdxConstant(
Idx - NumElems, dl)));
3527 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3534 Results.push_back(TLI.expandVectorSplice(
Node, DAG));
3538 EVT OpTy =
Node->getOperand(0).getValueType();
3539 if (
Node->getConstantOperandVal(1)) {
3543 TLI.getShiftAmountTy(
3544 Node->getOperand(0).getValueType(),
3545 DAG.getDataLayout())));
3550 Node->getOperand(0));
3558 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3559 Results.push_back(DAG.getCopyFromReg(
Node->getOperand(0), dl, SP,
3560 Node->getValueType(0)));
3563 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
3570 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3571 Results.push_back(DAG.getCopyToReg(
Node->getOperand(0), dl, SP,
3572 Node->getOperand(1)));
3578 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3593 TLI.expandIS_FPCLASS(
Node->getValueType(0),
Node->getOperand(0),
3604 switch (
Node->getOpcode()) {
3611 Tmp1 =
Node->getOperand(0);
3612 Tmp2 =
Node->getOperand(1);
3613 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3619 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG))
3625 if (
SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(
Node, DAG))
3631 EVT VT =
Node->getValueType(0);
3637 SDVTList VTs = DAG.getVTList(VT, VT);
3647 EVT VT =
Node->getValueType(0);
3651 if (TLI.getLibcallName(LC))
3657 Results.push_back(Expanded.getValue(1));
3666 if (TLI.getLibcallName(LC))
3671 Results.push_back(Expanded.getValue(1));
3679 if (
Node->getValueType(0) != MVT::f32) {
3691 if (
Node->getValueType(0) != MVT::f32) {
3696 {Node->getOperand(0), Node->getOperand(1)});
3698 {
Node->getValueType(0), MVT::Other},
3706 if (!TLI.useSoftFloat() &&
TM.Options.UnsafeFPMath) {
3708 MVT SVT =
Op.getSimpleValueType();
3709 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3715 DAG.getIntPtrConstant(0, dl,
true));
3726 DAG.shouldOptForSize()))
3727 Results.push_back(ExpandConstantFP(CFP,
true));
3732 Results.push_back(ExpandConstant(CP));
3736 EVT VT =
Node->getValueType(0);
3737 if (TLI.isOperationLegalOrCustom(
ISD::FADD, VT) &&
3738 TLI.isOperationLegalOrCustom(
ISD::FNEG, VT)) {
3747 EVT VT =
Node->getValueType(0);
3749 TLI.isOperationLegalOrCustom(
ISD::XOR, VT) &&
3750 "Don't know how to expand this subtraction!");
3751 Tmp1 = DAG.getNOT(dl,
Node->getOperand(1), VT);
3752 Tmp1 = DAG.
getNode(
ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3758 if (TLI.expandREM(
Node, Tmp1, DAG))
3765 EVT VT =
Node->getValueType(0);
3766 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3767 SDVTList VTs = DAG.getVTList(VT, VT);
3768 Tmp1 = DAG.
getNode(DivRemOpc, dl, VTs,
Node->getOperand(0),
3769 Node->getOperand(1));
3776 unsigned ExpandOpcode =
3778 EVT VT =
Node->getValueType(0);
3779 SDVTList VTs = DAG.getVTList(VT, VT);
3781 Tmp1 = DAG.
getNode(ExpandOpcode, dl, VTs,
Node->getOperand(0),
3782 Node->getOperand(1));
3790 MVT VT =
LHS.getSimpleValueType();
3791 unsigned MULHOpcode =
3794 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3796 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3802 assert(TLI.isTypeLegal(HalfType));
3803 if (TLI.expandMUL_LOHI(
Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3805 TargetLowering::MulExpansionKind::Always)) {
3806 for (
unsigned i = 0; i < 2; ++i) {
3809 SDValue Shift = DAG.getConstant(
3811 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3820 EVT VT =
Node->getValueType(0);
3821 SDVTList VTs = DAG.getVTList(VT, VT);
3827 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::SMUL_LOHI, VT);
3828 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::UMUL_LOHI, VT);
3829 bool HasMULHS = TLI.isOperationLegalOrCustom(
ISD::MULHS, VT);
3830 bool HasMULHU = TLI.isOperationLegalOrCustom(
ISD::MULHU, VT);
3831 unsigned OpToUse = 0;
3832 if (HasSMUL_LOHI && !HasMULHS) {
3834 }
else if (HasUMUL_LOHI && !HasMULHU) {
3836 }
else if (HasSMUL_LOHI) {
3838 }
else if (HasUMUL_LOHI) {
3842 Results.push_back(DAG.getNode(OpToUse, dl, VTs,
Node->getOperand(0),
3843 Node->getOperand(1)));
3851 TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
3852 TLI.isOperationLegalOrCustom(
ISD::OR, VT) &&
3853 TLI.expandMUL(
Node,
Lo,
Hi, HalfType, DAG,
3854 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3859 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3867 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG))
3872 if (
SDValue Expanded = TLI.expandROT(
Node,
true , DAG))
3879 Results.push_back(TLI.expandAddSubSat(
Node, DAG));
3889 Results.push_back(TLI.expandFixedPointMul(
Node, DAG));
3896 Node->getOperand(0),
3897 Node->getOperand(1),
3898 Node->getConstantOperandVal(2),
3921 EVT VT =
LHS.getValueType();
3925 EVT CarryType =
Node->getValueType(1);
3926 EVT SetCCType = getSetCCResultType(
Node->getValueType(0));
3928 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS,
CC);
3931 SDValue One = DAG.getConstant(1, dl, VT);
3933 DAG.
getNode(
ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3942 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero,
ISD::SETEQ)
3943 : DAG.getSetCC(dl, SetCCType, Sum, Zero,
ISD::SETEQ);
3945 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3951 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3957 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
3965 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
3973 if (TLI.expandMULO(
Node, Result, Overflow, DAG)) {
3985 DAG.getConstant(
PairTy.getSizeInBits() / 2, dl,
3986 TLI.getShiftAmountTy(
PairTy, DAG.getDataLayout())));
3991 Tmp1 =
Node->getOperand(0);
3992 Tmp2 =
Node->getOperand(1);
3993 Tmp3 =
Node->getOperand(2);
3997 cast<CondCodeSDNode>(Tmp1.
getOperand(2))->get());
3999 Tmp1 = DAG.getSelectCC(dl, Tmp1,
4010 int JTI = cast<JumpTableSDNode>(Table.
getNode())->getIndex();
4013 EVT PTy = TLI.getPointerTy(TD);
4015 unsigned EntrySize =
4016 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4023 Index = DAG.getNode(
4028 DAG.getConstant(EntrySize, dl,
Index.getValueType()));
4037 if (TLI.isJumpTableRelative()) {
4042 TLI.getPICJumpTableRelocBase(Table, DAG));
4045 Tmp1 = TLI.expandIndirectJTBranch(dl,
LD.getValue(1),
Addr, JTI, DAG);
4052 Tmp1 =
Node->getOperand(0);
4053 Tmp2 =
Node->getOperand(1);
4059 Node->getOperand(2));
4071 Node->getOperand(2));
4079 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
4084 unsigned Offset = IsStrict ? 1 : 0;
4093 bool Legalized = TLI.LegalizeSetCCCondCode(
4094 DAG,
Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4095 Chain, IsSignaling);
4103 {Chain, Tmp1, Tmp2, Tmp3},
Node->getFlags());
4107 {Tmp1, Tmp2, Tmp3, Mask, EVL},
Node->getFlags());
4109 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl,
Node->getValueType(0), Tmp1,
4110 Tmp2, Tmp3,
Node->getFlags());
4118 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->
getValueType(0));
4121 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->
getValueType(0));
4133 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
4138 EVT VT =
Node->getValueType(0);
4141 DAG.getBoolConstant(
true, dl, VT, Tmp1VT),
4142 DAG.getBoolConstant(
false, dl, VT, Tmp1VT), Tmp3);
4149 Tmp1 =
Node->getOperand(0);
4150 Tmp2 =
Node->getOperand(1);
4151 Tmp3 =
Node->getOperand(2);
4152 Tmp4 =
Node->getOperand(3);
4153 EVT VT =
Node->getValueType(0);
4163 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4165 EVT CCVT = getSetCCResultType(CmpVT);
4168 DAG.getSelect(dl, VT,
Cond, Tmp3, Tmp4,
Node->getFlags()));
4173 bool Legalized =
false;
4181 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4191 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4197 Legalized = TLI.LegalizeSetCCCondCode(
4201 assert(Legalized &&
"Can't legalize SELECT_CC with legal condition!");
4212 Tmp1, Tmp2, Tmp3, Tmp4,
CC);
4217 Tmp2, Tmp3, Tmp4,
CC);
4227 Tmp1 =
Node->getOperand(0);
4228 Tmp2 =
Node->getOperand(2);
4229 Tmp3 =
Node->getOperand(3);
4230 Tmp4 =
Node->getOperand(1);
4232 bool Legalized = TLI.LegalizeSetCCCondCode(
4233 DAG, getSetCCResultType(Tmp2.
getValueType()), Tmp2, Tmp3, Tmp4,
4236 assert(Legalized &&
"Can't legalize BR_CC with legal condition!");
4241 assert(!NeedInvert &&
"Don't know how to invert BR_CC!");
4244 Tmp4, Tmp2, Tmp3,
Node->getOperand(4));
4249 Tmp2, Tmp3,
Node->getOperand(4));
4264 EVT VT =
Node->getValueType(0);
4270 for (
unsigned Idx = 0;
Idx < NumElem;
Idx++) {
4273 Node->getOperand(0), DAG.getVectorIdxConstant(
Idx, dl));
4276 Node->getOperand(1), DAG.getVectorIdxConstant(
Idx, dl));
4300 Results.push_back(TLI.expandVecReduce(
Node, DAG));
4302 case ISD::VP_CTTZ_ELTS:
4303 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
4304 Results.push_back(TLI.expandVPCTTZElements(
Node, DAG));
4326 if (!TLI.isStrictFPEnabled() &&
Results.
empty() &&
Node->isStrictFPOpcode()) {
4332 switch (
Node->getOpcode()) {
4334 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4335 Node->getValueType(0))
4336 == TargetLowering::Legal)
4340 if (TLI.getStrictFPOperationAction(
4343 if (TLI.getStrictFPOperationAction(
4347 EVT VT =
Node->getValueType(0);
4351 {Node->getOperand(0), Node->getOperand(1), Neg},
4366 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4367 Node->getOperand(1).getValueType())
4368 == TargetLowering::Legal)
4385void SelectionDAGLegalize::ConvertNodeToLibcall(
SDNode *
Node) {
4390 unsigned Opc =
Node->getOpcode();
4399 .setChain(
Node->getOperand(0))
4402 DAG.getExternalSymbol(
"__sync_synchronize",
4403 TLI.getPointerTy(DAG.getDataLayout())),
4406 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4408 Results.push_back(CallResult.second);
4430 EVT RetVT =
Node->getValueType(0);
4433 if (TLI.getLibcallName(LC)) {
4440 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4441 "Unexpected atomic op or value type!");
4445 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4448 Node->getOperand(0));
4450 Results.push_back(Tmp.second);
4458 .setChain(
Node->getOperand(0))
4460 DAG.getExternalSymbol(
4461 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4463 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4465 Results.push_back(CallResult.second);
4473 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4474 DAG, RTLIB::CLEAR_CACHE, MVT::isVoid, {StartVal, EndVal}, CallOptions,
4476 Results.push_back(Tmp.second);
4481 ExpandFPLibCall(
Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4482 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4483 RTLIB::FMIN_PPCF128,
Results);
4490 ExpandFPLibCall(
Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4491 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4492 RTLIB::FMAX_PPCF128,
Results);
4496 ExpandFPLibCall(
Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4497 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4498 RTLIB::SQRT_PPCF128,
Results);
4501 ExpandFPLibCall(
Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4502 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4503 RTLIB::CBRT_PPCF128,
Results);
4507 ExpandFPLibCall(
Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4508 RTLIB::SIN_F80, RTLIB::SIN_F128,
4513 ExpandFPLibCall(
Node, RTLIB::COS_F32, RTLIB::COS_F64,
4514 RTLIB::COS_F80, RTLIB::COS_F128,
4519 ExpandFPLibCall(
Node, RTLIB::TAN_F32, RTLIB::TAN_F64, RTLIB::TAN_F80,
4520 RTLIB::TAN_F128, RTLIB::TAN_PPCF128,
Results);
4528 ExpandFPLibCall(
Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4529 RTLIB::LOG_F128, RTLIB::LOG_PPCF128,
Results);
4533 ExpandFPLibCall(
Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4534 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128,
Results);
4538 ExpandFPLibCall(
Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4539 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128,
Results);
4543 ExpandFPLibCall(
Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4544 RTLIB::EXP_F128, RTLIB::EXP_PPCF128,
Results);
4548 ExpandFPLibCall(
Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4549 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128,
Results);
4552 ExpandFPLibCall(
Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4553 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128,
Results);
4557 ExpandFPLibCall(
Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4558 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4559 RTLIB::TRUNC_PPCF128,
Results);
4563 ExpandFPLibCall(
Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4564 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4565 RTLIB::FLOOR_PPCF128,
Results);
4569 ExpandFPLibCall(
Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4570 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4571 RTLIB::CEIL_PPCF128,
Results);
4575 ExpandFPLibCall(
Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4576 RTLIB::RINT_F80, RTLIB::RINT_F128,
4577 RTLIB::RINT_PPCF128,
Results);
4581 ExpandFPLibCall(
Node, RTLIB::NEARBYINT_F32,
4582 RTLIB::NEARBYINT_F64,
4583 RTLIB::NEARBYINT_F80,
4584 RTLIB::NEARBYINT_F128,
4585 RTLIB::NEARBYINT_PPCF128,
Results);
4589 ExpandFPLibCall(
Node, RTLIB::ROUND_F32,
4593 RTLIB::ROUND_PPCF128,
Results);
4597 ExpandFPLibCall(
Node, RTLIB::ROUNDEVEN_F32,
4598 RTLIB::ROUNDEVEN_F64,
4599 RTLIB::ROUNDEVEN_F80,
4600 RTLIB::ROUNDEVEN_F128,
4601 RTLIB::ROUNDEVEN_PPCF128,
Results);
4605 ExpandFPLibCall(
Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4606 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128,
Results);
4615 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
4616 if (!TLI.getLibcallName(LC)) {
4618 if (
Node->isStrictFPOpcode()) {
4621 {
Node->getValueType(0),
Node->getValueType(1)},
4622 {
Node->getOperand(0),
Node->getOperand(2)});
4625 {
Node->getValueType(0),
Node->getValueType(1)},
4632 Node->getOperand(1));
4634 Node->getValueType(0),
4639 unsigned Offset =
Node->isStrictFPOpcode() ? 1 : 0;
4640 bool ExponentHasSizeOfInt =
4641 DAG.getLibInfo().getIntSize() ==
4642 Node->getOperand(1 +
Offset).getValueType().getSizeInBits();
4643 if (!ExponentHasSizeOfInt) {
4646 DAG.getContext()->emitError(
"POWI exponent does not match sizeof(int)");
4647 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
4655 ExpandFPLibCall(
Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4656 RTLIB::POW_F128, RTLIB::POW_PPCF128,
Results);
4660 ExpandArgFPLibCall(
Node, RTLIB::LROUND_F32,
4661 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4663 RTLIB::LROUND_PPCF128,
Results);
4667 ExpandArgFPLibCall(
Node, RTLIB::LLROUND_F32,
4668 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4669 RTLIB::LLROUND_F128,
4670 RTLIB::LLROUND_PPCF128,
Results);
4674 ExpandArgFPLibCall(
Node, RTLIB::LRINT_F32,
4675 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4677 RTLIB::LRINT_PPCF128,
Results);
4681 ExpandArgFPLibCall(
Node, RTLIB::LLRINT_F32,
4682 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4684 RTLIB::LLRINT_PPCF128,
Results);
4688 ExpandFPLibCall(
Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4689 RTLIB::DIV_F80, RTLIB::DIV_F128,
4694 ExpandFPLibCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
4695 RTLIB::REM_F80, RTLIB::REM_F128,
4700 ExpandFPLibCall(
Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4701 RTLIB::FMA_F80, RTLIB::FMA_F128,
4706 ExpandFPLibCall(
Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4707 RTLIB::ADD_F80, RTLIB::ADD_F128,
4712 ExpandFPLibCall(
Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4713 RTLIB::MUL_F80, RTLIB::MUL_F128,
4717 if (
Node->getValueType(0) == MVT::f32) {
4718 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32,
Node,
false).first);
4722 if (
Node->getValueType(0) == MVT::f32) {
4724 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4725 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32,
Node->getOperand(1),
4728 Results.push_back(Tmp.second);
4732 if (
Node->getValueType(0) == MVT::f32) {
4734 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4735 DAG, RTLIB::FPEXT_F16_F32, MVT::f32,
Node->getOperand(1), CallOptions,
4738 Results.push_back(Tmp.second);
4745 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16");
4746 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4752 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_bf16");
4753 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4761 bool IsStrict =
Node->isStrictFPOpcode();
4764 EVT SVT =
Node->getOperand(IsStrict ? 1 : 0).getValueType();
4765 EVT RVT =
Node->getValueType(0);
4773 for (
unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4774 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4782 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4787 NVT,
Node->getOperand(IsStrict ? 1 : 0));
4790 std::pair<SDValue, SDValue> Tmp =
4791 TLI.makeLibCall(DAG, LC, RVT,
Op, CallOptions, dl, Chain);
4794 Results.push_back(Tmp.second);
4802 bool IsStrict =
Node->isStrictFPOpcode();
4807 EVT SVT =
Op.getValueType();
4808 EVT RVT =
Node->getValueType(0);
4816 for (
unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4817 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4825 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4829 std::pair<SDValue, SDValue> Tmp =
4830 TLI.makeLibCall(DAG, LC, NVT,
Op, CallOptions, dl, Chain);
4835 Results.push_back(Tmp.second);
4846 bool IsStrict =
Node->isStrictFPOpcode();
4849 EVT VT =
Node->getValueType(0);
4850 assert(cast<ConstantSDNode>(
Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4851 "Unable to expand as libcall if it is not normal rounding");
4854 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4857 std::pair<SDValue, SDValue> Tmp =
4858 TLI.makeLibCall(DAG, LC, VT,
Op, CallOptions,
SDLoc(
Node), Chain);
4861 Results.push_back(Tmp.second);
4867 Node->getValueType(0)),
4868 Node,
false).first);
4881 Node->getValueType(0));
4883 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4886 std::pair<SDValue, SDValue> Tmp =
4887 TLI.makeLibCall(DAG, LC,
Node->getValueType(0),
Node->getOperand(1),
4890 Results.push_back(Tmp.second);
4895 ExpandFPLibCall(
Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4896 RTLIB::SUB_F80, RTLIB::SUB_F128,
4902 RTLIB::SREM_I16, RTLIB::SREM_I32,
4903 RTLIB::SREM_I64, RTLIB::SREM_I128));
4908 RTLIB::UREM_I16, RTLIB::UREM_I32,
4909 RTLIB::UREM_I64, RTLIB::UREM_I128));
4914 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4915 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4920 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4921 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4931 RTLIB::MUL_I16, RTLIB::MUL_I32,
4932 RTLIB::MUL_I64, RTLIB::MUL_I128));
4935 switch (
Node->getSimpleValueType(0).SimpleTy) {
4939 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32,
Node,
false).first);
4942 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64,
Node,
false).first);
4945 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128,
Node,
false).first);
4952 SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
4955 DAG.makeStateFunctionCall(RTLIB::FESETENV,
Ptr, Chain, dl));
4962 DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl));
4969 DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl));
4975 EVT ModeVT =
Node->getValueType(0);
4977 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4978 SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr,
4979 Node->getOperand(0), dl);
4981 ModeVT, dl, Chain, StackPtr,
4991 EVT ModeVT =
Mode.getValueType();
4993 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4994 SDValue StInst = DAG.getStore(
4995 Node->getOperand(0), dl, Mode, StackPtr,
4998 DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl));
5006 EVT PtrTy = TLI.getPointerTy(
DL);
5008 Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
5009 Node->getOperand(0), dl));
5016 LLVM_DEBUG(
dbgs() <<
"Successfully converted node to libcall\n");
5025 MVT EltVT,
MVT NewEltVT) {
5027 MVT MidVT = OldEltsPerNewElt == 1
5034void SelectionDAGLegalize::PromoteNode(
SDNode *
Node) {
5037 MVT OVT =
Node->getSimpleValueType(0);
5043 OVT =
Node->getOperand(0).getSimpleValueType();
5050 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5051 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5052 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5053 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5054 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5055 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5056 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5057 OVT =
Node->getOperand(1).getSimpleValueType();
5060 OVT =
Node->getOperand(2).getSimpleValueType();
5061 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), OVT);
5063 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5064 switch (
Node->getOpcode()) {
5077 unsigned NewOpc =
Node->getOpcode();
5085 DAG.getConstant(TopBit, dl, NVT));
5090 Tmp1 = DAG.
getNode(NewOpc, dl, NVT, Tmp1);
5104 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5107 DAG.getConstant(DiffBits, dl,
5108 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
5121 Results.push_back(PromoteLegalFP_TO_INT_SAT(
Node, dl));
5138 &&
"VAARG promotion is supported only for vectors or integer types");
5143 Tmp1 = DAG.getVAArg(NVT, dl, Chain,
Ptr,
Node->getOperand(2),
5144 Node->getConstantOperandVal(3));
5147 Tmp2 = DAG.
getNode(TruncOp, dl, OVT, Tmp1);
5151 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), Tmp2);
5152 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
5154 UpdatedNodes->insert(Tmp2.
getNode());
5155 UpdatedNodes->insert(Chain.
getNode());
5172 unsigned ExtOp, TruncOp;
5179 switch (
Node->getOpcode()) {
5195 if (TLI.isSExtCheaperThanZExt(OVT, NVT))
5204 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5205 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5207 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5208 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
5216 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5217 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5220 auto &
DL = DAG.getDataLayout();
5224 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(
DL, NVT)));
5230 unsigned ExtOp, TruncOp;
5231 if (
Node->getValueType(0).isVector() ||
5235 }
else if (
Node->getValueType(0).isInteger()) {
5242 Tmp1 =
Node->getOperand(0);
5244 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5245 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5247 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5250 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1);
5252 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1,
5253 DAG.getIntPtrConstant(0, dl));
5265 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5274 Node->getOperand(2));
5282 MVT CVT =
Node->getSimpleValueType(0);
5283 assert(CVT == OVT &&
"not handled");
5291 if (TLI.isCondCodeLegal(CCCode, CVT)) {
5292 Tmp1 =
Node->getOperand(0);
5293 Tmp2 =
Node->getOperand(1);
5295 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5296 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5299 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5300 Tmp4 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5310 DAG.getIntPtrConstant(0, dl,
true));
5322 TLI.isSExtCheaperThanZExt(
Node->getOperand(0).getValueType(), NVT))
5327 if (
Node->isStrictFPOpcode()) {
5329 std::tie(Tmp1, std::ignore) =
5330 DAG.getStrictFPExtendOrRound(
Node->getOperand(1), InChain, dl, NVT);
5331 std::tie(Tmp2, std::ignore) =
5332 DAG.getStrictFPExtendOrRound(
Node->getOperand(2), InChain, dl, NVT);
5334 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
5335 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
5336 Results.push_back(DAG.getNode(
Node->getOpcode(), dl, VTs,
5337 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5342 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5343 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5345 Tmp2,
Node->getOperand(2),
Node->getFlags()));
5352 cast<CondCodeSDNode>(
Node->getOperand(1))->get();
5355 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5356 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5358 Node->getOperand(0),
Node->getOperand(1),
5359 Tmp1, Tmp2,
Node->getOperand(4)));
5374 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
5378 DAG.getIntPtrConstant(0, dl,
true)));
5389 {
Node->getOperand(0),
Node->getOperand(1)});
5391 {
Node->getOperand(0),
Node->getOperand(2)});
5394 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5395 {Tmp3, Tmp1, Tmp2});
5397 {Tmp1.
getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5407 DAG.getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5408 DAG.getIntPtrConstant(0, dl,
true)));
5412 {
Node->getOperand(0),
Node->getOperand(1)});
5414 {
Node->getOperand(0),
Node->getOperand(2)});
5416 {
Node->getOperand(0),
Node->getOperand(3)});
5419 Tmp4 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5420 {Tmp4, Tmp1, Tmp2, Tmp3});
5422 {Tmp4.
getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5430 Tmp2 =
Node->getOperand(1);
5431 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5441 DAG.getIntPtrConstant(isTrunc, dl,
true)));
5446 {
Node->getOperand(0),
Node->getOperand(1)});
5447 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5450 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5460 DAG.getIntPtrConstant(0, dl,
true)));
5486 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5489 DAG.getIntPtrConstant(0, dl,
true)));
5508 {
Node->getOperand(0),
Node->getOperand(1)});
5509 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5512 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5527 "Invalid promote type for build_vector");
5533 for (
unsigned I = 0, E =
Node->getNumOperands();
I != E; ++
I) {
5562 "Invalid promote type for extract_vector_elt");
5569 EVT IdxVT =
Idx.getValueType();
5571 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
5577 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5578 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5586 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
5608 "Invalid promote type for insert_vector_elt");
5616 EVT IdxVT =
Idx.getValueType();
5619 SDValue Factor = DAG.getConstant(NewEltsPerOldElt,
SDLoc(), IdxVT);
5626 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5627 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5631 CastVal, IdxOffset);
5634 NewVec, Elt, InEltIdx);
5674 "unexpected promotion type");
5676 "unexpected atomic_swap with illegal type");
5700 "unexpected promotion type");
5702 "unexpected atomic_load with illegal type");
5713 MVT ScalarType =
Scalar.getSimpleValueType();
5717 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5722 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5725 DAG.getIntPtrConstant(0, dl,
true)));
5728 case ISD::VP_REDUCE_FADD:
5729 case ISD::VP_REDUCE_FMUL:
5730 case ISD::VP_REDUCE_FMAX:
5731 case ISD::VP_REDUCE_FMIN:
5732 case ISD::VP_REDUCE_FMAXIMUM:
5733 case ISD::VP_REDUCE_FMINIMUM:
5734 case ISD::VP_REDUCE_SEQ_FADD:
5760 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes);
5767 bool AnyLegalized =
false;
5778 if (LegalizedNodes.
insert(
N).second) {
5779 AnyLegalized =
true;
5800 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes, &UpdatedNodes);
5807 return LegalizedNodes.
count(
N);
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
static MaybeAlign getAlign(Value *Ptr)
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI)
Return true if sincos libcall is available.
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
static MachineMemOperand * getStackAlignedMMO(SDValue StackPtr, MachineFunction &MF, bool isObjectScalable)
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
mir Rename Register Operands
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This is an SDNode representing atomic operations.
const SDValue & getBasePtr() const
const SDValue & getVal() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
const BasicBlock & back() const
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOStore
The memory access writes data.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetLowering & getTargetLoweringInfo() const
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
allnodes_const_iterator allnodes_end() const
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const DataLayout & getDataLayout() const
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
LLVMContext * getContext() const
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
Information about stack frame layout on the target.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
constexpr ScalarTy getFixedValue() const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ FRAME_TO_ARGS_OFFSET
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
@ RESET_FPENV
Set floating-point environment to default state.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ GLOBAL_OFFSET_TABLE
The address of the GOT.
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ SET_FPENV_MEM
Sets the current floating point environment.
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
@ Undef
Value of the register doesn't matter.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
int32_t ExponentType
A signed type to represent a floating point numbers unbiased exponent.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)