58#define DEBUG_TYPE "legalizedag"
64struct FloatSignAsInt {
87class SelectionDAGLegalize {
99 EVT getSetCCResultType(
EVT VT)
const {
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
173 void getSignAsIntValue(FloatSignAsInt &State,
const SDLoc &
DL,
175 SDValue modifySignAsInt(
const FloatSignAsInt &State,
const SDLoc &
DL,
215 dbgs() <<
" with: "; New->dump(&DAG));
218 "Replacing one node with another that produces a different number "
222 UpdatedNodes->
insert(New);
228 dbgs() <<
" with: "; New->dump(&DAG));
232 UpdatedNodes->
insert(New.getNode());
233 ReplacedNode(Old.getNode());
240 for (
unsigned i = 0, e = Old->
getNumValues(); i != e; ++i) {
244 UpdatedNodes->
insert(New[i].getNode());
251 dbgs() <<
" with: "; New->dump(&DAG));
255 UpdatedNodes->
insert(New.getNode());
256 ReplacedNode(Old.getNode());
266 bool isObjectScalable) {
268 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();
274 ObjectSize, MFI.getObjectAlign(FI));
281SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
286 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
288 assert(NumEltsGrowth &&
"Cannot promote to vector type with fewer elts!");
290 if (NumEltsGrowth == 1)
291 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
294 for (
unsigned i = 0; i != NumMaskElts; ++i) {
296 for (
unsigned j = 0;
j != NumEltsGrowth; ++
j) {
303 assert(NewMask.
size() == NumDestElts &&
"Non-integer NumEltsGrowth?");
304 assert(TLI.isShuffleMaskLegal(NewMask, NVT) &&
"Shuffle not legal?");
305 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
324 assert((VT == MVT::f64 || VT == MVT::f32) &&
"Invalid type expansion");
326 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
336 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
342 TLI.ShouldShrinkFPConstant(OrigVT)) {
345 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
353 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
354 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
363 OrigVT, dl, DAG.getEntryNode(), CPIdx,
371 EVT VT =
CP->getValueType(0);
372 SDValue CPIdx = DAG.getConstantPool(
CP->getConstantIntValue(),
373 TLI.getPointerTy(DAG.getDataLayout()));
374 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
376 VT, dl, DAG.getEntryNode(), CPIdx,
385SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(
SDValue Vec,
399 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
403 DAG.getEntryNode(), dl, Vec, StackPtr,
406 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT,
Idx);
409 Ch = DAG.getTruncStore(
410 Ch, dl, Val, StackPtr2,
414 DAG.getMachineFunction(), SPFI));
435 for (
unsigned i = 0; i != NumElts; ++i)
436 ShufOps.
push_back(i != InsertPos->getZExtValue() ? i : NumElts);
438 return DAG.getVectorShuffle(Vec.
getValueType(), dl, Vec, ScVec, ShufOps);
441 return PerformInsertVectorEltInMemory(Vec, Val,
Idx, dl);
468 TLI.isTypeLegal(MVT::i32)) {
470 bitcastToAPInt().zextOrTrunc(32),
471 SDLoc(CFP), MVT::i32);
472 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
473 ST->getOriginalAlign(), MMOFlags, AAInfo);
479 if (TLI.isTypeLegal(MVT::i64)) {
481 zextOrTrunc(64),
SDLoc(CFP), MVT::i64);
482 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
483 ST->getOriginalAlign(), MMOFlags, AAInfo);
486 if (TLI.isTypeLegal(MVT::i32) && !
ST->isVolatile()) {
493 if (DAG.getDataLayout().isBigEndian())
496 Lo = DAG.getStore(Chain, dl,
Lo,
Ptr,
ST->getPointerInfo(),
497 ST->getOriginalAlign(), MMOFlags, AAInfo);
499 Hi = DAG.getStore(Chain, dl,
Hi,
Ptr,
500 ST->getPointerInfo().getWithOffset(4),
501 ST->getOriginalAlign(), MMOFlags, AAInfo);
510void SelectionDAGLegalize::LegalizeStoreOps(
SDNode *
Node) {
519 if (!
ST->isTruncatingStore()) {
521 if (
SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
522 ReplaceNode(ST, OptStore);
527 MVT VT =
Value.getSimpleValueType();
528 switch (TLI.getOperationAction(
ISD::STORE, VT)) {
530 case TargetLowering::Legal: {
533 EVT MemVT =
ST->getMemoryVT();
535 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
536 *
ST->getMemOperand())) {
539 ReplaceNode(
SDValue(ST, 0), Result);
544 case TargetLowering::Custom: {
551 case TargetLowering::Promote: {
554 "Can only promote stores to same size type");
557 ST->getOriginalAlign(), MMOFlags, AAInfo);
567 EVT StVT =
ST->getMemoryVT();
570 auto &
DL = DAG.getDataLayout();
572 if (StWidth != StSize) {
577 Value = DAG.getZeroExtendInReg(
Value, dl, StVT);
579 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), NVT,
580 ST->getOriginalAlign(), MMOFlags, AAInfo);
586 unsigned LogStWidth =
Log2_32(StWidthBits);
588 unsigned RoundWidth = 1 << LogStWidth;
589 assert(RoundWidth < StWidthBits);
590 unsigned ExtraWidth = StWidthBits - RoundWidth;
591 assert(ExtraWidth < RoundWidth);
592 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
593 "Store size not an integral number of bytes!");
597 unsigned IncrementSize;
599 if (
DL.isLittleEndian()) {
602 Lo = DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(),
603 RoundVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
606 IncrementSize = RoundWidth / 8;
611 DAG.getConstant(RoundWidth, dl,
612 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
613 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
614 ST->getPointerInfo().getWithOffset(IncrementSize),
615 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
622 DAG.getConstant(ExtraWidth, dl,
623 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
624 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
ST->getPointerInfo(), RoundVT,
625 ST->getOriginalAlign(), MMOFlags, AAInfo);
628 IncrementSize = RoundWidth / 8;
630 DAG.getConstant(IncrementSize, dl,
631 Ptr.getValueType()));
633 ST->getPointerInfo().getWithOffset(IncrementSize),
634 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
641 switch (TLI.getTruncStoreAction(
ST->getValue().getValueType(), StVT)) {
643 case TargetLowering::Legal: {
644 EVT MemVT =
ST->getMemoryVT();
647 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
648 *
ST->getMemOperand())) {
650 ReplaceNode(
SDValue(ST, 0), Result);
654 case TargetLowering::Custom: {
660 case TargetLowering::Expand:
662 "Vector Stores are handled in LegalizeVectorOps");
667 if (TLI.isTypeLegal(StVT)) {
670 ST->getOriginalAlign(), MMOFlags, AAInfo);
675 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
678 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), StVT,
679 ST->getOriginalAlign(), MMOFlags, AAInfo);
688void SelectionDAGLegalize::LegalizeLoadOps(
SDNode *
Node) {
697 LLVM_DEBUG(
dbgs() <<
"Legalizing non-extending load operation\n");
698 MVT VT =
Node->getSimpleValueType(0);
702 switch (TLI.getOperationAction(
Node->getOpcode(), VT)) {
704 case TargetLowering::Legal: {
705 EVT MemVT =
LD->getMemoryVT();
709 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
710 *
LD->getMemOperand())) {
711 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
715 case TargetLowering::Custom:
716 if (
SDValue Res = TLI.LowerOperation(RVal, DAG)) {
722 case TargetLowering::Promote: {
723 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
725 "Can only promote loads to same size type");
727 SDValue Res = DAG.getLoad(NVT, dl, Chain,
Ptr,
LD->getMemOperand());
735 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), RVal);
736 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), RChain);
738 UpdatedNodes->insert(RVal.
getNode());
739 UpdatedNodes->insert(RChain.
getNode());
747 EVT SrcVT =
LD->getMemoryVT();
761 TLI.getLoadExtAction(ExtType,
Node->getValueType(0), MVT::i1) ==
762 TargetLowering::Promote)) {
776 Chain,
Ptr,
LD->getPointerInfo(), NVT,
777 LD->getOriginalAlign(), MMOFlags, AAInfo);
785 Result, DAG.getValueType(SrcVT));
789 Result.getValueType(), Result,
790 DAG.getValueType(SrcVT));
798 unsigned LogSrcWidth =
Log2_32(SrcWidthBits);
800 unsigned RoundWidth = 1 << LogSrcWidth;
801 assert(RoundWidth < SrcWidthBits);
802 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
803 assert(ExtraWidth < RoundWidth);
804 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
805 "Load size not an integral number of bytes!");
809 unsigned IncrementSize;
810 auto &
DL = DAG.getDataLayout();
812 if (
DL.isLittleEndian()) {
816 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
820 IncrementSize = RoundWidth / 8;
823 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
824 LD->getPointerInfo().getWithOffset(IncrementSize),
825 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
835 DAG.getConstant(RoundWidth, dl,
836 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
844 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
845 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
849 IncrementSize = RoundWidth / 8;
853 LD->getPointerInfo().getWithOffset(IncrementSize),
854 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
864 DAG.getConstant(ExtraWidth, dl,
865 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
873 bool isCustom =
false;
874 switch (TLI.getLoadExtAction(ExtType,
Node->getValueType(0),
877 case TargetLowering::Custom:
880 case TargetLowering::Legal:
892 EVT MemVT =
LD->getMemoryVT();
894 if (!TLI.allowsMemoryAccess(*DAG.getContext(),
DL, MemVT,
895 *
LD->getMemOperand())) {
896 std::tie(
Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
901 case TargetLowering::Expand: {
902 EVT DestVT =
Node->getValueType(0);
908 (TLI.isTypeLegal(SrcVT) ||
909 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
916 SrcVT,
LD->getMemOperand());
919 Value = DAG.getNode(ExtendOp, dl,
Node->getValueType(0), Load);
920 Chain =
Load.getValue(1);
929 if (SVT == MVT::f16 || SVT == MVT::bf16) {
935 Ptr, ISrcVT,
LD->getMemOperand());
939 Chain =
Result.getValue(1);
945 "Vector Loads are handled in LegalizeVectorOps");
952 "EXTLOAD should always be supported!");
956 Node->getValueType(0),
958 LD->getMemOperand());
963 Result, DAG.getValueType(SrcVT));
965 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
967 Chain =
Result.getValue(1);
978 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
980 UpdatedNodes->insert(
Value.getNode());
981 UpdatedNodes->insert(Chain.
getNode());
988void SelectionDAGLegalize::LegalizeOp(
SDNode *
Node) {
997 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
998 assert(TLI.getTypeAction(*DAG.getContext(),
Node->getValueType(i)) ==
999 TargetLowering::TypeLegal &&
1000 "Unexpected illegal type!");
1003 assert((TLI.getTypeAction(*DAG.getContext(),
Op.getValueType()) ==
1004 TargetLowering::TypeLegal ||
1007 "Unexpected illegal type!");
1012 bool SimpleFinishLegalizing =
true;
1013 switch (
Node->getOpcode()) {
1018 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
1021 Action = TLI.getOperationAction(
Node->getOpcode(),
1022 Node->getValueType(0));
1025 Action = TLI.getOperationAction(
Node->getOpcode(),
1026 Node->getValueType(0));
1027 if (Action != TargetLowering::Promote)
1028 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
1032 Action = TLI.getOperationAction(
Node->getOpcode(),
1033 Node->getOperand(1).getValueType());
1044 Action = TLI.getOperationAction(
Node->getOpcode(),
1045 Node->getOperand(0).getValueType());
1058 Action = TLI.getOperationAction(
Node->getOpcode(),
1059 Node->getOperand(1).getValueType());
1062 EVT InnerType = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1063 Action = TLI.getOperationAction(
Node->getOpcode(), InnerType);
1067 Action = TLI.getOperationAction(
Node->getOpcode(),
1068 Node->getOperand(1).getValueType());
1077 unsigned Opc =
Node->getOpcode();
1082 : (Opc ==
ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1084 unsigned CompareOperand = Opc ==
ISD::BR_CC ? 2
1088 MVT OpVT =
Node->getOperand(CompareOperand).getSimpleValueType();
1090 cast<CondCodeSDNode>(
Node->getOperand(CCOperand))->get();
1091 Action = TLI.getCondCodeAction(CCCode, OpVT);
1092 if (Action == TargetLowering::Legal) {
1094 Action = TLI.getOperationAction(
Node->getOpcode(),
1095 Node->getValueType(0));
1097 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
1105 SimpleFinishLegalizing =
false;
1112 SimpleFinishLegalizing =
false;
1125 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1126 if (Action == TargetLowering::Legal)
1127 Action = TargetLowering::Expand;
1137 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1138 if (Action == TargetLowering::Legal)
1139 Action = TargetLowering::Custom;
1145 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::i64);
1152 Action = TargetLowering::Legal;
1155 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1156 if (Action == TargetLowering::Expand) {
1160 Node->getOperand(0));
1167 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1168 if (Action == TargetLowering::Expand) {
1172 Node->getOperand(0));
1186 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1196 unsigned Scale =
Node->getConstantOperandVal(2);
1197 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
1198 Node->getValueType(0), Scale);
1202 Action = TLI.getOperationAction(
Node->getOpcode(),
1203 cast<MaskedScatterSDNode>(
Node)->getValue().getValueType());
1206 Action = TLI.getOperationAction(
Node->getOpcode(),
1207 cast<MaskedStoreSDNode>(
Node)->getValue().getValueType());
1209 case ISD::VP_SCATTER:
1210 Action = TLI.getOperationAction(
1212 cast<VPScatterSDNode>(
Node)->getValue().getValueType());
1215 Action = TLI.getOperationAction(
1217 cast<VPStoreSDNode>(
Node)->getValue().getValueType());
1219 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1220 Action = TLI.getOperationAction(
1222 cast<VPStridedStoreSDNode>(
Node)->getValue().getValueType());
1240 Action = TLI.getOperationAction(
1241 Node->getOpcode(),
Node->getOperand(0).getValueType());
1245 case ISD::VP_REDUCE_FADD:
1246 case ISD::VP_REDUCE_FMUL:
1247 case ISD::VP_REDUCE_ADD:
1248 case ISD::VP_REDUCE_MUL:
1249 case ISD::VP_REDUCE_AND:
1250 case ISD::VP_REDUCE_OR:
1251 case ISD::VP_REDUCE_XOR:
1252 case ISD::VP_REDUCE_SMAX:
1253 case ISD::VP_REDUCE_SMIN:
1254 case ISD::VP_REDUCE_UMAX:
1255 case ISD::VP_REDUCE_UMIN:
1256 case ISD::VP_REDUCE_FMAX:
1257 case ISD::VP_REDUCE_FMIN:
1258 case ISD::VP_REDUCE_SEQ_FADD:
1259 case ISD::VP_REDUCE_SEQ_FMUL:
1260 Action = TLI.getOperationAction(
1261 Node->getOpcode(),
Node->getOperand(1).getValueType());
1265 Action = TLI.getCustomOperationAction(*
Node);
1267 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1272 if (SimpleFinishLegalizing) {
1274 switch (
Node->getOpcode()) {
1293 NewNode = DAG.UpdateNodeOperands(
Node, Op0, SAO);
1313 NewNode = DAG.UpdateNodeOperands(
Node, Op0, Op1, SAO);
1319 if (NewNode !=
Node) {
1320 ReplaceNode(
Node, NewNode);
1324 case TargetLowering::Legal:
1327 case TargetLowering::Custom:
1335 if (
Node->getNumValues() == 1) {
1339 Node->getValueType(0) == MVT::Glue) &&
1340 "Type mismatch for custom legalized operation");
1348 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i) {
1352 Node->getValueType(i) == MVT::Glue) &&
1353 "Type mismatch for custom legalized operation");
1357 ReplaceNode(
Node, ResultVals.
data());
1362 case TargetLowering::Expand:
1363 if (ExpandNode(
Node))
1366 case TargetLowering::LibCall:
1367 ConvertNodeToLibcall(
Node);
1369 case TargetLowering::Promote:
1375 switch (
Node->getOpcode()) {
1388 return LegalizeLoadOps(
Node);
1390 return LegalizeStoreOps(
Node);
1394SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(
SDValue Op) {
1414 if (
ST->isIndexed() ||
ST->isTruncatingStore() ||
1415 ST->getValue() != Vec)
1420 if (!
ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1429 ST->hasPredecessor(
Op.getNode()))
1442 StackPtr = DAG.CreateStackTemporary(VecVT);
1445 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO);
1449 Align ElementAlignment =
1450 std::min(cast<StoreSDNode>(Ch)->
getAlign(),
1451 DAG.getDataLayout().getPrefTypeAlign(
1452 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1454 if (
Op.getValueType().isVector()) {
1455 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1456 Op.getValueType(),
Idx);
1457 NewLoad = DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr,
1460 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1461 NewLoad = DAG.getExtLoad(
ISD::EXTLOAD, dl,
Op.getValueType(), Ch, StackPtr,
1467 DAG.ReplaceAllUsesOfValueWith(Ch,
SDValue(NewLoad.
getNode(), 1));
1473 NewLoadOperands[0] = Ch;
1475 SDValue(DAG.UpdateNodeOperands(NewLoad.
getNode(), NewLoadOperands), 0);
1479SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(
SDValue Op) {
1480 assert(
Op.getValueType().isVector() &&
"Non-vector insert subvector!");
1491 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1496 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1500 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT,
Idx);
1504 Ch, dl, Part, SubStackPtr,
1508 return DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1514 "Unexpected opcode!");
1520 EVT VT =
Node->getValueType(0);
1522 :
Node->getOperand(0).getValueType();
1524 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1525 int FI = cast<FrameIndexSDNode>(FIPtr.
getNode())->getIndex();
1532 assert(TypeByteSize > 0 &&
"Vector element type too small for stack store!");
1536 bool Truncate = isa<BuildVectorSDNode>(
Node) &&
1537 MemVT.
bitsLT(
Node->getOperand(0).getValueType());
1540 for (
unsigned i = 0, e =
Node->getNumOperands(); i != e; ++i) {
1542 if (
Node->getOperand(i).isUndef())
continue;
1544 unsigned Offset = TypeByteSize*i;
1550 Stores.
push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1554 Stores.
push_back(DAG.getStore(DAG.getEntryNode(), dl,
Node->getOperand(i),
1559 if (!Stores.
empty())
1562 StoreChain = DAG.getEntryNode();
1565 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1571void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1574 EVT FloatVT =
Value.getValueType();
1576 State.FloatVT = FloatVT;
1579 if (TLI.isTypeLegal(IVT)) {
1582 State.SignBit = NumBits - 1;
1588 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1591 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1596 State.Chain = DAG.getStore(DAG.getEntryNode(),
DL,
Value, State.FloatPtr,
1597 State.FloatPointerInfo);
1604 State.IntPointerInfo = State.FloatPointerInfo;
1607 unsigned ByteOffset = (NumBits / 8) - 1;
1614 State.IntPtr = IntPtr;
1615 State.IntValue = DAG.getExtLoad(
ISD::EXTLOAD,
DL, LoadTy, State.Chain, IntPtr,
1616 State.IntPointerInfo, MVT::i8);
1623SDValue SelectionDAGLegalize::modifySignAsInt(
const FloatSignAsInt &State,
1630 SDValue Chain = DAG.getTruncStore(State.Chain,
DL, NewIntValue, State.IntPtr,
1631 State.IntPointerInfo, MVT::i8);
1632 return DAG.getLoad(State.FloatVT,
DL, Chain, State.FloatPtr,
1633 State.FloatPointerInfo);
1642 FloatSignAsInt SignAsInt;
1643 getSignAsIntValue(SignAsInt,
DL, Sign);
1645 EVT IntVT = SignAsInt.IntValue.getValueType();
1646 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1652 if (TLI.isOperationLegalOrCustom(
ISD::FABS, FloatVT) &&
1653 TLI.isOperationLegalOrCustom(
ISD::FNEG, FloatVT)) {
1656 SDValue Cond = DAG.getSetCC(
DL, getSetCCResultType(IntVT), SignBit,
1658 return DAG.getSelect(
DL, FloatVT,
Cond, NegValue, AbsValue);
1662 FloatSignAsInt MagAsInt;
1663 getSignAsIntValue(MagAsInt,
DL, Mag);
1664 EVT MagVT = MagAsInt.IntValue.getValueType();
1665 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask,
DL, MagVT);
1670 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1671 EVT ShiftVT = IntVT;
1677 if (ShiftAmount > 0) {
1678 SDValue ShiftCnst = DAG.getConstant(ShiftAmount,
DL, ShiftVT);
1680 }
else if (ShiftAmount < 0) {
1681 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount,
DL, ShiftVT);
1691 return modifySignAsInt(MagAsInt,
DL, CopiedSign);
1697 FloatSignAsInt SignAsInt;
1698 getSignAsIntValue(SignAsInt,
DL,
Node->getOperand(0));
1699 EVT IntVT = SignAsInt.IntValue.getValueType();
1702 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1707 return modifySignAsInt(SignAsInt,
DL, SignFlip);
1715 EVT FloatVT =
Value.getValueType();
1722 FloatSignAsInt ValueAsInt;
1723 getSignAsIntValue(ValueAsInt,
DL,
Value);
1724 EVT IntVT = ValueAsInt.IntValue.getValueType();
1725 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask,
DL, IntVT);
1728 return modifySignAsInt(ValueAsInt,
DL, ClearedSign);
1731void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(
SDNode*
Node,
1733 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1734 assert(SPReg &&
"Target cannot require DYNAMIC_STACKALLOC expansion and"
1735 " not tell us which reg is the stack pointer!");
1737 EVT VT =
Node->getValueType(0);
1745 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1748 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1750 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1758 if (Alignment > StackAlign)
1760 DAG.getConstant(-Alignment.
value(), dl, VT));
1761 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);
1763 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0,
SDValue(), dl);
1775 return EmitStackConvert(
SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1783 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1786 if ((SrcVT.
bitsGT(SlotVT) &&
1787 !TLI.isTruncStoreLegalOrCustom(
SrcOp.getValueType(), SlotVT)) ||
1788 (SlotVT.
bitsLT(DestVT) &&
1789 !TLI.isLoadExtLegalOrCustom(
ISD::EXTLOAD, DestVT, SlotVT)))
1793 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1794 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1806 if (SrcVT.
bitsGT(SlotVT))
1807 Store = DAG.getTruncStore(Chain, dl,
SrcOp, FIPtr, PtrInfo,
1811 Store = DAG.getStore(Chain, dl,
SrcOp, FIPtr, PtrInfo, SrcAlign);
1815 if (SlotVT.
bitsEq(DestVT))
1816 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1819 return DAG.getExtLoad(
ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1832 SDValue Ch = DAG.getTruncStore(
1833 DAG.getEntryNode(), dl,
Node->getOperand(0), StackPtr,
1835 Node->getValueType(0).getVectorElementType());
1837 Node->getValueType(0), dl, Ch, StackPtr,
1844 unsigned NumElems =
Node->getNumOperands();
1846 EVT VT =
Node->getValueType(0);
1858 for (
unsigned i = 0; i < NumElems; ++i) {
1869 while (IntermedVals.
size() > 2) {
1870 NewIntermedVals.
clear();
1871 for (
unsigned i = 0, e = (IntermedVals.
size() & ~1u); i < e; i += 2) {
1877 FinalIndices.
reserve(IntermedVals[i].second.
size() +
1878 IntermedVals[i+1].second.
size());
1881 for (
unsigned j = 0, f = IntermedVals[i].second.
size(); j != f;
1884 FinalIndices.
push_back(IntermedVals[i].second[j]);
1886 for (
unsigned j = 0, f = IntermedVals[i+1].second.
size(); j != f;
1888 ShuffleVec[k] = NumElems + j;
1889 FinalIndices.
push_back(IntermedVals[i+1].second[j]);
1895 IntermedVals[i+1].first,
1900 std::make_pair(Shuffle, std::move(FinalIndices)));
1905 if ((IntermedVals.
size() & 1) != 0)
1908 IntermedVals.
swap(NewIntermedVals);
1912 "Invalid number of intermediate vectors");
1913 SDValue Vec1 = IntermedVals[0].first;
1915 if (IntermedVals.
size() > 1)
1916 Vec2 = IntermedVals[1].first;
1921 for (
unsigned i = 0, e = IntermedVals[0].second.
size(); i != e; ++i)
1922 ShuffleVec[IntermedVals[0].second[i]] = i;
1923 for (
unsigned i = 0, e = IntermedVals[1].second.
size(); i != e; ++i)
1924 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1938 unsigned NumElems =
Node->getNumOperands();
1941 EVT VT =
Node->getValueType(0);
1942 EVT OpVT =
Node->getOperand(0).getValueType();
1947 bool isOnlyLowElement =
true;
1948 bool MoreThanTwoValues =
false;
1950 for (
unsigned i = 0; i < NumElems; ++i) {
1955 isOnlyLowElement =
false;
1956 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1961 }
else if (!Value2.
getNode()) {
1964 }
else if (V != Value1 && V != Value2) {
1965 MoreThanTwoValues =
true;
1970 return DAG.getUNDEF(VT);
1972 if (isOnlyLowElement)
1978 for (
unsigned i = 0, e = NumElems; i !=
e; ++i) {
1980 dyn_cast<ConstantFPSDNode>(
Node->getOperand(i))) {
1983 dyn_cast<ConstantSDNode>(
Node->getOperand(i))) {
2002 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
2003 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2005 VT, dl, DAG.getEntryNode(), CPIdx,
2011 for (
unsigned i = 0; i < NumElems; ++i) {
2012 if (
Node->getOperand(i).isUndef())
2017 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.
size())) {
2018 if (!MoreThanTwoValues) {
2020 for (
unsigned i = 0; i < NumElems; ++i) {
2024 ShuffleVec[i] =
V == Value1 ? 0 : NumElems;
2026 if (TLI.isShuffleMaskLegal(ShuffleVec,
Node->getValueType(0))) {
2033 Vec2 = DAG.getUNDEF(VT);
2036 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2046 return ExpandVectorBuildThroughStack(
Node);
2051 EVT VT =
Node->getValueType(0);
2054 return DAG.getSplatBuildVector(VT,
DL, SplatVal);
2065 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2066 TLI.getPointerTy(DAG.getDataLayout()));
2068 EVT RetVT =
Node->getValueType(0);
2075 SDValue InChain = DAG.getEntryNode();
2080 const Function &
F = DAG.getMachineFunction().getFunction();
2082 TLI.isInTailCallPosition(DAG,
Node, TCChain) &&
2083 (
RetTy ==
F.getReturnType() ||
F.getReturnType()->isVoidTy());
2088 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT,
isSigned);
2091 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2093 .setTailCall(isTailCall)
2094 .setSExtResult(signExtend)
2095 .setZExtResult(!signExtend)
2096 .setIsPostTypeLegalization(
true);
2098 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2103 return {DAG.getRoot(), DAG.getRoot()};
2115 EVT ArgVT =
Op.getValueType();
2119 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT,
isSigned);
2120 Entry.IsZExt = !Entry.IsSExt;
2121 Args.push_back(Entry);
2124 return ExpandLibCall(LC,
Node, std::move(Args),
isSigned);
2127void SelectionDAGLegalize::ExpandFrexpLibCall(
2130 EVT VT =
Node->getValueType(0);
2131 EVT ExpVT =
Node->getValueType(1);
2139 FPArgEntry.
Node = FPOp;
2140 FPArgEntry.
Ty = ArgTy;
2142 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2144 PtrArgEntry.
Node = StackSlot;
2145 PtrArgEntry.
Ty = PointerType::get(*DAG.getContext(),
2146 DAG.getDataLayout().getAllocaAddrSpace());
2151 auto [
Call, Chain] = ExpandLibCall(LC,
Node, std::move(Args),
false);
2155 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2159 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2161 LoadExp.
getValue(1), DAG.getRoot());
2162 DAG.setRoot(OutputChain);
2168void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2171 if (LC == RTLIB::UNKNOWN_LIBCALL)
2174 if (
Node->isStrictFPOpcode()) {
2175 EVT RetVT =
Node->getValueType(0);
2179 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2182 Node->getOperand(0));
2184 Results.push_back(Tmp.second);
2186 SDValue Tmp = ExpandLibCall(LC,
Node,
false).first;
2192void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2200 Call_F32, Call_F64, Call_F80,
2201 Call_F128, Call_PPCF128);
2212 switch (
Node->getSimpleValueType(0).SimpleTy) {
2214 case MVT::i8: LC = Call_I8;
break;
2215 case MVT::i16: LC = Call_I16;
break;
2216 case MVT::i32: LC = Call_I32;
break;
2217 case MVT::i64: LC = Call_I64;
break;
2218 case MVT::i128: LC = Call_I128;
break;
2225void SelectionDAGLegalize::ExpandArgFPLibCall(
SDNode*
Node,
2232 EVT InVT =
Node->getOperand(
Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2234 Call_F32, Call_F64, Call_F80,
2235 Call_F128, Call_PPCF128);
2241SelectionDAGLegalize::ExpandDivRemLibCall(
SDNode *
Node,
2243 unsigned Opcode =
Node->getOpcode();
2247 switch (
Node->getSimpleValueType(0).SimpleTy) {
2249 case MVT::i8: LC=
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
2250 case MVT::i16: LC=
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
2251 case MVT::i32: LC=
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
2252 case MVT::i64: LC=
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
2253 case MVT::i128: LC=
isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
2259 SDValue InChain = DAG.getEntryNode();
2261 EVT RetVT =
Node->getValueType(0);
2267 EVT ArgVT =
Op.getValueType();
2273 Args.push_back(Entry);
2277 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2279 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2282 Args.push_back(Entry);
2284 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2285 TLI.getPointerTy(DAG.getDataLayout()));
2291 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2296 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2308 switch (
Node->getSimpleValueType(0).SimpleTy) {
2310 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2311 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2312 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2313 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2314 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2337SelectionDAGLegalize::ExpandSinCosLibCall(
SDNode *
Node,
2340 switch (
Node->getSimpleValueType(0).SimpleTy) {
2342 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2343 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2344 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2345 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2346 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2352 SDValue InChain = DAG.getEntryNode();
2354 EVT RetVT =
Node->getValueType(0);
2361 Entry.Node =
Node->getOperand(0);
2363 Entry.IsSExt =
false;
2364 Entry.IsZExt =
false;
2365 Args.push_back(Entry);
2368 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2369 Entry.Node = SinPtr;
2370 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2371 Entry.IsSExt =
false;
2372 Entry.IsZExt =
false;
2373 Args.push_back(Entry);
2376 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2377 Entry.Node = CosPtr;
2378 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2379 Entry.IsSExt =
false;
2380 Entry.IsZExt =
false;
2381 Args.push_back(Entry);
2383 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2384 TLI.getPointerTy(DAG.getDataLayout()));
2388 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2389 TLI.getLibcallCallingConv(LC),
Type::getVoidTy(*DAG.getContext()), Callee,
2392 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2402 EVT VT =
Node->getValueType(0);
2405 EVT ExpVT =
N.getValueType();
2407 if (AsIntVT ==
EVT())
2420 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2425 const int Precision = APFloat::semanticsPrecision(FltSem);
2427 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2428 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2430 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2432 const APFloat One(FltSem,
"1.0");
2433 APFloat ScaleUpK =
scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2437 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2446 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2452 DAG.getSetCC(dl, SetCCVT,
N, DoubleMaxExp,
ISD::SETUGT);
2454 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2466 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2467 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2472 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2477 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2481 SDValue ScaleDownTwice = DAG.getSetCC(
2482 dl, SetCCVT,
N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2494 DAG.getNode(
ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small,
X));
2498 DAG.getNode(
ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small,
N));
2503 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2504 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2507 ExponentShiftAmt, NUW_NSW);
2509 return DAG.getNode(
ISD::FMUL, dl, VT, NewX, AsFP);
2516 EVT ExpVT =
Node->getValueType(1);
2518 if (AsIntVT ==
EVT())
2523 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2540 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2544 SDValue SmallestNormalizedInt = DAG.getConstant(
2550 DAG.getConstant(
APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2556 FractSignMaskVal.
setBit(BitSize - 1);
2559 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2561 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2563 const APFloat One(FltSem,
"1.0");
2567 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2569 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2573 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2579 SDValue AddNegSmallestNormal =
2581 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2585 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt,
ISD::SETULT);
2587 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2602 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2605 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2608 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2615 SDValue FPHalf = DAG.getConstant(
Half.bitcastToAPInt(), dl, AsIntVT);
2628 return DAG.getMergeValues({Result0, Result1}, dl);
2639 EVT DestVT =
Node->getValueType(0);
2641 unsigned OpNo =
Node->isStrictFPOpcode() ? 1 : 0;
2647 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2648 (DestVT.
bitsLE(MVT::f64) ||
2652 LLVM_DEBUG(
dbgs() <<
"32-bit [signed|unsigned] integer to float/double "
2656 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2663 DAG.getConstant(0x80000000u, dl, MVT::i32));
2666 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2669 if (DAG.getDataLayout().isBigEndian())
2672 SDValue MemChain = DAG.getEntryNode();
2675 SDValue Store1 = DAG.getStore(MemChain, dl,
Lo, StackSlot,
2688 SDValue Bias = DAG.getConstantFP(
2689 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2690 : llvm::bit_cast<double>(0x4330000000000000ULL),
2695 if (
Node->isStrictFPOpcode()) {
2697 {
Node->getOperand(0), Load, Bias});
2700 std::pair<SDValue, SDValue> ResultPair;
2702 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2703 Result = ResultPair.first;
2704 Chain = ResultPair.second;
2710 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2719 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2720 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2721 LLVM_DEBUG(
dbgs() <<
"Converting unsigned i32/i64 to f32/f64\n");
2736 EVT SetCCVT = getSetCCResultType(SrcVT);
2738 SDValue SignBitTest = DAG.getSetCC(
2739 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT),
ISD::SETLT);
2741 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2742 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2744 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2749 if (
Node->isStrictFPOpcode()) {
2752 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest,
Or, Op0);
2754 {
Node->getOperand(0), InCvt });
2762 Flags.setNoFPExcept(
Node->getFlags().hasNoFPExcept());
2763 Fast->setFlags(Flags);
2764 Flags.setNoFPExcept(
true);
2772 return DAG.getSelect(dl, DestVT, SignBitTest, Slow,
Fast);
2776 if (!TLI.isOperationLegalOrCustom(
2783 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2785 "Cannot perform lossless SINT_TO_FP!");
2788 if (
Node->isStrictFPOpcode()) {
2790 {
Node->getOperand(0), Op0 });
2794 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2797 Four = DAG.getIntPtrConstant(4, dl);
2798 SDValue CstOffset = DAG.getSelect(dl,
Zero.getValueType(),
2799 SignSet, Four, Zero);
2808 case MVT::i8 : FF = 0x43800000ULL;
break;
2809 case MVT::i16: FF = 0x47800000ULL;
break;
2810 case MVT::i32: FF = 0x4F800000ULL;
break;
2811 case MVT::i64: FF = 0x5F800000ULL;
break;
2813 if (DAG.getDataLayout().isLittleEndian())
2815 Constant *FudgeFactor = ConstantInt::get(
2819 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2820 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2824 if (DestVT == MVT::f32)
2825 FudgeInReg = DAG.getLoad(
2826 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2835 LegalizeOp(
Load.getNode());
2839 if (
Node->isStrictFPOpcode()) {
2841 { Tmp1.
getValue(1), Tmp1, FudgeInReg });
2842 Chain =
Result.getValue(1);
2846 return DAG.getNode(
ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2854void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2856 bool IsStrict =
N->isStrictFPOpcode();
2859 EVT DestVT =
N->getValueType(0);
2860 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2867 unsigned OpToUse = 0;
2875 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2883 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2895 DAG.
getNode(OpToUse, dl, {DestVT, MVT::Other},
2898 dl, NewInTy, LegalOp)});
2905 DAG.getNode(OpToUse, dl, DestVT,
2907 dl, NewInTy, LegalOp)));
2915void SelectionDAGLegalize::PromoteLegalFP_TO_INT(
SDNode *
N,
const SDLoc &dl,
2917 bool IsStrict =
N->isStrictFPOpcode();
2920 EVT DestVT =
N->getValueType(0);
2921 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2923 EVT NewOutTy = DestVT;
2925 unsigned OpToUse = 0;
2935 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2940 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2949 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2950 Operation = DAG.getNode(OpToUse, dl, VTs,
N->getOperand(0), LegalOp);
2952 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2967 unsigned Opcode =
Node->getOpcode();
2970 EVT NewOutTy =
Node->getValueType(0);
2975 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2982 Node->getOperand(1));
2988 EVT VT =
Op.getValueType();
2989 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2994 if (TLI.isOperationLegalOrPromote(
ISD::CTPOP, VT)) {
3000 DAG.getConstant(1ULL << (--i), dl, ShVT));
3005 return DAG.getNode(
ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
3008bool SelectionDAGLegalize::ExpandNode(
SDNode *
Node) {
3012 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3014 switch (
Node->getOpcode()) {
3016 if ((Tmp1 = TLI.expandABS(
Node, DAG)))
3021 if ((Tmp1 = TLI.expandABD(
Node, DAG)))
3025 if ((Tmp1 = TLI.expandCTPOP(
Node, DAG)))
3030 if ((Tmp1 = TLI.expandCTLZ(
Node, DAG)))
3035 if ((Tmp1 = TLI.expandCTTZ(
Node, DAG)))
3039 if ((Tmp1 = TLI.expandBITREVERSE(
Node, DAG)))
3043 if ((Tmp1 = TLI.expandBSWAP(
Node, DAG)))
3047 Results.push_back(ExpandPARITY(
Node->getOperand(0), dl));
3052 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3055 SDValue CfaArg = DAG.getSExtOrTrunc(
Node->getOperand(0), dl,
3056 TLI.getPointerTy(DAG.getDataLayout()));
3064 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3070 Results.push_back(DAG.getConstant(1, dl,
Node->getValueType(0)));
3087 DAG.getConstant(0, dl,
Node->getValueType(0)));
3093 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3099 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3100 SDValue Swap = DAG.getAtomicCmpSwap(
3102 Node->getOperand(0),
Node->getOperand(1), Zero, Zero,
3103 cast<AtomicSDNode>(
Node)->getMemOperand());
3112 Node->getOperand(0),
Node->getOperand(2),
Node->getOperand(1),
3113 cast<AtomicSDNode>(
Node)->getMemOperand());
3121 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3122 SDValue Res = DAG.getAtomicCmpSwap(
3124 Node->getOperand(0),
Node->getOperand(1),
Node->getOperand(2),
3125 Node->getOperand(3), cast<MemSDNode>(
Node)->getMemOperand());
3131 EVT AtomicType = cast<AtomicSDNode>(
Node)->getMemoryVT();
3132 EVT OuterType =
Node->getValueType(0);
3133 switch (TLI.getExtendForAtomicOps()) {
3136 DAG.getValueType(AtomicType));
3138 Node->getOperand(2), DAG.getValueType(AtomicType));
3143 DAG.getValueType(AtomicType));
3144 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3148 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3149 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3165 EVT VT =
Node->getValueType(0);
3169 cast<VTSDNode>(
RHS->getOperand(1))->getVT() == AN->
getMemoryVT())
3170 RHS =
RHS->getOperand(0);
3174 Node->getOperand(0),
Node->getOperand(1),
3184 for (
unsigned i = 0; i <
Node->getNumValues(); i++)
3188 EVT VT =
Node->getValueType(0);
3190 Results.push_back(DAG.getConstant(0, dl, VT));
3193 Results.push_back(DAG.getConstantFP(0, dl, VT));
3200 if (TLI.isStrictFPEnabled())
3204 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3205 Node->getValueType(0))
3206 == TargetLowering::Legal)
3210 if ((Tmp1 = EmitStackConvert(
Node->getOperand(1),
Node->getValueType(0),
3211 Node->getValueType(0), dl,
3212 Node->getOperand(0)))) {
3214 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_ROUND node\n");
3219 if ((Tmp1 = TLI.expandFP_ROUND(
Node, DAG))) {
3227 if ((Tmp1 = EmitStackConvert(
Node->getOperand(0),
Node->getValueType(0),
3228 Node->getValueType(0), dl)))
3234 if (TLI.isStrictFPEnabled())
3238 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3239 Node->getValueType(0))
3240 == TargetLowering::Legal)
3244 if ((Tmp1 = EmitStackConvert(
3245 Node->getOperand(1),
Node->getOperand(1).getValueType(),
3246 Node->getValueType(0), dl,
Node->getOperand(0)))) {
3248 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_EXTEND node\n");
3254 EVT SrcVT =
Op.getValueType();
3255 EVT DstVT =
Node->getValueType(0);
3261 if ((Tmp1 = EmitStackConvert(
Op, SrcVT, DstVT, dl)))
3271 if (
Op.getValueType() == MVT::bf16) {
3275 Op = DAG.getAnyExtOrTrunc(
Op, dl, MVT::i32);
3279 DAG.getConstant(16, dl,
3280 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3283 if (
Node->getValueType(0) != MVT::f32)
3290 if (
Op.getValueType() != MVT::f32)
3292 DAG.getIntPtrConstant(0, dl,
true));
3294 if (!DAG.isKnownNeverSNaN(
Op)) {
3299 DAG.getConstant(16, dl,
3300 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3303 if (
Node->getValueType(0) == MVT::bf16) {
3307 Op = DAG.getAnyExtOrTrunc(
Op, dl,
Node->getValueType(0));
3313 EVT ExtraVT = cast<VTSDNode>(
Node->getOperand(1))->getVT();
3314 EVT VT =
Node->getValueType(0);
3324 SDValue One = DAG.getConstant(1, dl, VT);
3334 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3337 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3339 Node->getOperand(0), ShiftCst);
3346 if (TLI.expandUINT_TO_FP(
Node, Tmp1, Tmp2, DAG)) {
3348 if (
Node->isStrictFPOpcode())
3355 if ((Tmp1 = ExpandLegalINT_TO_FP(
Node, Tmp2))) {
3357 if (
Node->isStrictFPOpcode())
3362 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG))
3366 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG)) {
3368 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_SINT node\n");
3373 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG))
3377 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG)) {
3379 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node,1), Tmp2);
3382 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_UINT node\n");
3388 Results.push_back(TLI.expandFP_TO_INT_SAT(
Node, DAG));
3398 if (
Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3401 Node->getOperand(0));
3403 Tmp1 = ExpandExtractFromVectorThroughStack(
SDValue(
Node, 0));
3413 Results.push_back(ExpandVectorBuildThroughStack(
Node));
3419 Results.push_back(ExpandINSERT_VECTOR_ELT(
Node->getOperand(0),
3420 Node->getOperand(1),
3421 Node->getOperand(2), dl));
3427 EVT VT =
Node->getValueType(0);
3431 if (!TLI.isTypeLegal(EltVT)) {
3432 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3437 if (NewEltVT.
bitsLT(EltVT)) {
3453 unsigned int factor =
3461 for (
unsigned fi = 0; fi < factor; ++fi)
3465 for (
unsigned fi = 0; fi < factor; ++fi)
3476 for (
unsigned i = 0; i != NumElems; ++i) {
3484 DAG.getVectorIdxConstant(
Idx, dl)));
3488 DAG.getVectorIdxConstant(
Idx - NumElems, dl)));
3491 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3498 Results.push_back(TLI.expandVectorSplice(
Node, DAG));
3502 EVT OpTy =
Node->getOperand(0).getValueType();
3503 if (
Node->getConstantOperandVal(1)) {
3507 TLI.getShiftAmountTy(
3508 Node->getOperand(0).getValueType(),
3509 DAG.getDataLayout())));
3514 Node->getOperand(0));
3522 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3523 Results.push_back(DAG.getCopyFromReg(
Node->getOperand(0), dl, SP,
3524 Node->getValueType(0)));
3527 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
3534 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3535 Results.push_back(DAG.getCopyToReg(
Node->getOperand(0), dl, SP,
3536 Node->getOperand(1)));
3542 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3557 TLI.expandIS_FPCLASS(
Node->getValueType(0),
Node->getOperand(0),
3568 switch (
Node->getOpcode()) {
3575 Tmp1 =
Node->getOperand(0);
3576 Tmp2 =
Node->getOperand(1);
3577 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3583 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG))
3589 EVT VT =
Node->getValueType(0);
3595 SDVTList VTs = DAG.getVTList(VT, VT);
3605 EVT VT =
Node->getValueType(0);
3609 if (TLI.getLibcallName(LC))
3615 Results.push_back(Expanded.getValue(1));
3624 if (TLI.getLibcallName(LC))
3629 Results.push_back(Expanded.getValue(1));
3637 if (
Node->getValueType(0) != MVT::f32) {
3649 if (
Node->getValueType(0) != MVT::f32) {
3654 {Node->getOperand(0), Node->getOperand(1)});
3656 {
Node->getValueType(0), MVT::Other},
3664 if (!TLI.useSoftFloat() &&
TM.Options.UnsafeFPMath) {
3666 MVT SVT =
Op.getSimpleValueType();
3667 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3673 DAG.getIntPtrConstant(0, dl,
true));
3684 DAG.shouldOptForSize()))
3685 Results.push_back(ExpandConstantFP(CFP,
true));
3690 Results.push_back(ExpandConstant(CP));
3694 EVT VT =
Node->getValueType(0);
3695 if (TLI.isOperationLegalOrCustom(
ISD::FADD, VT) &&
3696 TLI.isOperationLegalOrCustom(
ISD::FNEG, VT)) {
3705 EVT VT =
Node->getValueType(0);
3707 TLI.isOperationLegalOrCustom(
ISD::XOR, VT) &&
3708 "Don't know how to expand this subtraction!");
3709 Tmp1 = DAG.getNOT(dl,
Node->getOperand(1), VT);
3710 Tmp1 = DAG.
getNode(
ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3716 if (TLI.expandREM(
Node, Tmp1, DAG))
3723 EVT VT =
Node->getValueType(0);
3724 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3725 SDVTList VTs = DAG.getVTList(VT, VT);
3726 Tmp1 = DAG.
getNode(DivRemOpc, dl, VTs,
Node->getOperand(0),
3727 Node->getOperand(1));
3734 unsigned ExpandOpcode =
3736 EVT VT =
Node->getValueType(0);
3737 SDVTList VTs = DAG.getVTList(VT, VT);
3739 Tmp1 = DAG.
getNode(ExpandOpcode, dl, VTs,
Node->getOperand(0),
3740 Node->getOperand(1));
3748 MVT VT =
LHS.getSimpleValueType();
3749 unsigned MULHOpcode =
3752 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3754 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3760 assert(TLI.isTypeLegal(HalfType));
3761 if (TLI.expandMUL_LOHI(
Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3763 TargetLowering::MulExpansionKind::Always)) {
3764 for (
unsigned i = 0; i < 2; ++i) {
3767 SDValue Shift = DAG.getConstant(
3769 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3778 EVT VT =
Node->getValueType(0);
3779 SDVTList VTs = DAG.getVTList(VT, VT);
3785 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::SMUL_LOHI, VT);
3786 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::UMUL_LOHI, VT);
3787 bool HasMULHS = TLI.isOperationLegalOrCustom(
ISD::MULHS, VT);
3788 bool HasMULHU = TLI.isOperationLegalOrCustom(
ISD::MULHU, VT);
3789 unsigned OpToUse = 0;
3790 if (HasSMUL_LOHI && !HasMULHS) {
3792 }
else if (HasUMUL_LOHI && !HasMULHU) {
3794 }
else if (HasSMUL_LOHI) {
3796 }
else if (HasUMUL_LOHI) {
3800 Results.push_back(DAG.getNode(OpToUse, dl, VTs,
Node->getOperand(0),
3801 Node->getOperand(1)));
3809 TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
3810 TLI.isOperationLegalOrCustom(
ISD::OR, VT) &&
3811 TLI.expandMUL(
Node,
Lo,
Hi, HalfType, DAG,
3812 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3817 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3825 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG))
3830 if (
SDValue Expanded = TLI.expandROT(
Node,
true , DAG))
3837 Results.push_back(TLI.expandAddSubSat(
Node, DAG));
3847 Results.push_back(TLI.expandFixedPointMul(
Node, DAG));
3854 Node->getOperand(0),
3855 Node->getOperand(1),
3856 Node->getConstantOperandVal(2),
3879 EVT VT =
LHS.getValueType();
3883 EVT CarryType =
Node->getValueType(1);
3884 EVT SetCCType = getSetCCResultType(
Node->getValueType(0));
3886 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS,
CC);
3889 SDValue One = DAG.getConstant(1, dl, VT);
3891 DAG.
getNode(
ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3900 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero,
ISD::SETEQ)
3901 : DAG.getSetCC(dl, SetCCType, Sum, Zero,
ISD::SETEQ);
3903 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3909 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3915 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
3923 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
3931 if (TLI.expandMULO(
Node, Result, Overflow, DAG)) {
3943 DAG.getConstant(
PairTy.getSizeInBits() / 2, dl,
3944 TLI.getShiftAmountTy(
PairTy, DAG.getDataLayout())));
3949 Tmp1 =
Node->getOperand(0);
3950 Tmp2 =
Node->getOperand(1);
3951 Tmp3 =
Node->getOperand(2);
3955 cast<CondCodeSDNode>(Tmp1.
getOperand(2))->get());
3957 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3968 int JTI = cast<JumpTableSDNode>(Table.
getNode())->getIndex();
3971 EVT PTy = TLI.getPointerTy(TD);
3973 unsigned EntrySize =
3974 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3981 Index = DAG.getNode(
3986 DAG.getConstant(EntrySize, dl,
Index.getValueType()));
3995 if (TLI.isJumpTableRelative()) {
4000 TLI.getPICJumpTableRelocBase(Table, DAG));
4003 Tmp1 = TLI.expandIndirectJTBranch(dl,
LD.getValue(1),
Addr, JTI, DAG);
4010 Tmp1 =
Node->getOperand(0);
4011 Tmp2 =
Node->getOperand(1);
4017 Node->getOperand(2));
4029 Node->getOperand(2));
4037 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
4042 unsigned Offset = IsStrict ? 1 : 0;
4051 bool Legalized = TLI.LegalizeSetCCCondCode(
4052 DAG,
Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4053 Chain, IsSignaling);
4061 {Chain, Tmp1, Tmp2, Tmp3},
Node->getFlags());
4065 {Tmp1, Tmp2, Tmp3, Mask, EVL},
Node->getFlags());
4067 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl,
Node->getValueType(0), Tmp1,
4068 Tmp2, Tmp3,
Node->getFlags());
4076 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->
getValueType(0));
4079 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->
getValueType(0));
4091 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
4096 EVT VT =
Node->getValueType(0);
4099 DAG.getBoolConstant(
true, dl, VT, Tmp1VT),
4100 DAG.getBoolConstant(
false, dl, VT, Tmp1VT), Tmp3);
4107 Tmp1 =
Node->getOperand(0);
4108 Tmp2 =
Node->getOperand(1);
4109 Tmp3 =
Node->getOperand(2);
4110 Tmp4 =
Node->getOperand(3);
4111 EVT VT =
Node->getValueType(0);
4121 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4123 EVT CCVT = getSetCCResultType(CmpVT);
4125 Results.push_back(DAG.getSelect(dl, VT,
Cond, Tmp3, Tmp4));
4130 bool Legalized =
false;
4138 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4148 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4154 Legalized = TLI.LegalizeSetCCCondCode(
4158 assert(Legalized &&
"Can't legalize SELECT_CC with legal condition!");
4169 Tmp1, Tmp2, Tmp3, Tmp4,
CC);
4174 Tmp2, Tmp3, Tmp4,
CC);
4184 Tmp1 =
Node->getOperand(0);
4185 Tmp2 =
Node->getOperand(2);
4186 Tmp3 =
Node->getOperand(3);
4187 Tmp4 =
Node->getOperand(1);
4189 bool Legalized = TLI.LegalizeSetCCCondCode(
4190 DAG, getSetCCResultType(Tmp2.
getValueType()), Tmp2, Tmp3, Tmp4,
4193 assert(Legalized &&
"Can't legalize BR_CC with legal condition!");
4198 assert(!NeedInvert &&
"Don't know how to invert BR_CC!");
4201 Tmp4, Tmp2, Tmp3,
Node->getOperand(4));
4206 Tmp2, Tmp3,
Node->getOperand(4));
4221 EVT VT =
Node->getValueType(0);
4227 for (
unsigned Idx = 0;
Idx < NumElem;
Idx++) {
4230 Node->getOperand(0), DAG.getVectorIdxConstant(
Idx, dl));
4233 Node->getOperand(1), DAG.getVectorIdxConstant(
Idx, dl));
4257 Results.push_back(TLI.expandVecReduce(
Node, DAG));
4274 if (!TLI.isStrictFPEnabled() &&
Results.
empty() &&
Node->isStrictFPOpcode()) {
4280 switch (
Node->getOpcode()) {
4282 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4283 Node->getValueType(0))
4284 == TargetLowering::Legal)
4288 if (TLI.getStrictFPOperationAction(
4291 if (TLI.getStrictFPOperationAction(
4295 EVT VT =
Node->getValueType(0);
4299 {Node->getOperand(0), Node->getOperand(1), Neg},
4314 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4315 Node->getOperand(1).getValueType())
4316 == TargetLowering::Legal)
4333void SelectionDAGLegalize::ConvertNodeToLibcall(
SDNode *
Node) {
4338 unsigned Opc =
Node->getOpcode();
4347 .setChain(
Node->getOperand(0))
4350 DAG.getExternalSymbol(
"__sync_synchronize",
4351 TLI.getPointerTy(DAG.getDataLayout())),
4354 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4356 Results.push_back(CallResult.second);
4378 EVT RetVT =
Node->getValueType(0);
4381 if (TLI.getLibcallName(LC)) {
4388 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4389 "Unexpected atomic op or value type!");
4393 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4396 Node->getOperand(0));
4398 Results.push_back(Tmp.second);
4406 .setChain(
Node->getOperand(0))
4408 DAG.getExternalSymbol(
4409 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4411 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4413 Results.push_back(CallResult.second);
4418 ExpandFPLibCall(
Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4419 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4420 RTLIB::FMIN_PPCF128,
Results);
4427 ExpandFPLibCall(
Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4428 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4429 RTLIB::FMAX_PPCF128,
Results);
4433 ExpandFPLibCall(
Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4434 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4435 RTLIB::SQRT_PPCF128,
Results);
4438 ExpandFPLibCall(
Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4439 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4440 RTLIB::CBRT_PPCF128,
Results);
4444 ExpandFPLibCall(
Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4445 RTLIB::SIN_F80, RTLIB::SIN_F128,
4450 ExpandFPLibCall(
Node, RTLIB::COS_F32, RTLIB::COS_F64,
4451 RTLIB::COS_F80, RTLIB::COS_F128,
4460 ExpandFPLibCall(
Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4461 RTLIB::LOG_F128, RTLIB::LOG_PPCF128,
Results);
4465 ExpandFPLibCall(
Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4466 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128,
Results);
4470 ExpandFPLibCall(
Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4471 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128,
Results);
4475 ExpandFPLibCall(
Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4476 RTLIB::EXP_F128, RTLIB::EXP_PPCF128,
Results);
4480 ExpandFPLibCall(
Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4481 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128,
Results);
4484 ExpandFPLibCall(
Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4485 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128,
Results);
4489 ExpandFPLibCall(
Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4490 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4491 RTLIB::TRUNC_PPCF128,
Results);
4495 ExpandFPLibCall(
Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4496 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4497 RTLIB::FLOOR_PPCF128,
Results);
4501 ExpandFPLibCall(
Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4502 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4503 RTLIB::CEIL_PPCF128,
Results);
4507 ExpandFPLibCall(
Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4508 RTLIB::RINT_F80, RTLIB::RINT_F128,
4509 RTLIB::RINT_PPCF128,
Results);
4513 ExpandFPLibCall(
Node, RTLIB::NEARBYINT_F32,
4514 RTLIB::NEARBYINT_F64,
4515 RTLIB::NEARBYINT_F80,
4516 RTLIB::NEARBYINT_F128,
4517 RTLIB::NEARBYINT_PPCF128,
Results);
4521 ExpandFPLibCall(
Node, RTLIB::ROUND_F32,
4525 RTLIB::ROUND_PPCF128,
Results);
4529 ExpandFPLibCall(
Node, RTLIB::ROUNDEVEN_F32,
4530 RTLIB::ROUNDEVEN_F64,
4531 RTLIB::ROUNDEVEN_F80,
4532 RTLIB::ROUNDEVEN_F128,
4533 RTLIB::ROUNDEVEN_PPCF128,
Results);
4537 ExpandFPLibCall(
Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4538 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128,
Results);
4547 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
4548 if (!TLI.getLibcallName(LC)) {
4550 if (
Node->isStrictFPOpcode()) {
4553 {
Node->getValueType(0),
Node->getValueType(1)},
4554 {
Node->getOperand(0),
Node->getOperand(2)});
4557 {
Node->getValueType(0),
Node->getValueType(1)},
4564 Node->getOperand(1));
4566 Node->getValueType(0),
4571 unsigned Offset =
Node->isStrictFPOpcode() ? 1 : 0;
4572 bool ExponentHasSizeOfInt =
4573 DAG.getLibInfo().getIntSize() ==
4574 Node->getOperand(1 +
Offset).getValueType().getSizeInBits();
4575 if (!ExponentHasSizeOfInt) {
4578 DAG.getContext()->emitError(
"POWI exponent does not match sizeof(int)");
4579 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
4587 ExpandFPLibCall(
Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4588 RTLIB::POW_F128, RTLIB::POW_PPCF128,
Results);
4592 ExpandArgFPLibCall(
Node, RTLIB::LROUND_F32,
4593 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4595 RTLIB::LROUND_PPCF128,
Results);
4599 ExpandArgFPLibCall(
Node, RTLIB::LLROUND_F32,
4600 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4601 RTLIB::LLROUND_F128,
4602 RTLIB::LLROUND_PPCF128,
Results);
4606 ExpandArgFPLibCall(
Node, RTLIB::LRINT_F32,
4607 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4609 RTLIB::LRINT_PPCF128,
Results);
4613 ExpandArgFPLibCall(
Node, RTLIB::LLRINT_F32,
4614 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4616 RTLIB::LLRINT_PPCF128,
Results);
4620 ExpandFPLibCall(
Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4621 RTLIB::DIV_F80, RTLIB::DIV_F128,
4626 ExpandFPLibCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
4627 RTLIB::REM_F80, RTLIB::REM_F128,
4632 ExpandFPLibCall(
Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4633 RTLIB::FMA_F80, RTLIB::FMA_F128,
4638 ExpandFPLibCall(
Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4639 RTLIB::ADD_F80, RTLIB::ADD_F128,
4644 ExpandFPLibCall(
Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4645 RTLIB::MUL_F80, RTLIB::MUL_F128,
4649 if (
Node->getValueType(0) == MVT::f32) {
4650 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32,
Node,
false).first);
4654 if (
Node->getValueType(0) == MVT::f32) {
4656 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4657 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32,
Node->getOperand(1),
4660 Results.push_back(Tmp.second);
4664 if (
Node->getValueType(0) == MVT::f32) {
4666 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4667 DAG, RTLIB::FPEXT_F16_F32, MVT::f32,
Node->getOperand(1), CallOptions,
4670 Results.push_back(Tmp.second);
4677 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16");
4678 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4684 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_bf16");
4685 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4693 bool IsStrict =
Node->isStrictFPOpcode();
4696 EVT SVT =
Node->getOperand(IsStrict ? 1 : 0).getValueType();
4697 EVT RVT =
Node->getValueType(0);
4705 for (
unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4706 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4714 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4719 NVT,
Node->getOperand(IsStrict ? 1 : 0));
4722 std::pair<SDValue, SDValue> Tmp =
4723 TLI.makeLibCall(DAG, LC, RVT,
Op, CallOptions, dl, Chain);
4726 Results.push_back(Tmp.second);
4734 bool IsStrict =
Node->isStrictFPOpcode();
4739 EVT SVT =
Op.getValueType();
4740 EVT RVT =
Node->getValueType(0);
4748 for (
unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4749 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4757 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4761 std::pair<SDValue, SDValue> Tmp =
4762 TLI.makeLibCall(DAG, LC, NVT,
Op, CallOptions, dl, Chain);
4767 Results.push_back(Tmp.second);
4778 bool IsStrict =
Node->isStrictFPOpcode();
4781 EVT VT =
Node->getValueType(0);
4782 assert(cast<ConstantSDNode>(
Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4783 "Unable to expand as libcall if it is not normal rounding");
4786 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4789 std::pair<SDValue, SDValue> Tmp =
4790 TLI.makeLibCall(DAG, LC, VT,
Op, CallOptions,
SDLoc(
Node), Chain);
4793 Results.push_back(Tmp.second);
4799 Node->getValueType(0)),
4800 Node,
false).first);
4813 Node->getValueType(0));
4815 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4818 std::pair<SDValue, SDValue> Tmp =
4819 TLI.makeLibCall(DAG, LC,
Node->getValueType(0),
Node->getOperand(1),
4822 Results.push_back(Tmp.second);
4827 ExpandFPLibCall(
Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4828 RTLIB::SUB_F80, RTLIB::SUB_F128,
4834 RTLIB::SREM_I16, RTLIB::SREM_I32,
4835 RTLIB::SREM_I64, RTLIB::SREM_I128));
4840 RTLIB::UREM_I16, RTLIB::UREM_I32,
4841 RTLIB::UREM_I64, RTLIB::UREM_I128));
4846 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4847 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4852 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4853 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4863 RTLIB::MUL_I16, RTLIB::MUL_I32,
4864 RTLIB::MUL_I64, RTLIB::MUL_I128));
4867 switch (
Node->getSimpleValueType(0).SimpleTy) {
4871 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32,
Node,
false).first);
4874 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64,
Node,
false).first);
4877 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128,
Node,
false).first);
4884 SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
4887 DAG.makeStateFunctionCall(RTLIB::FESETENV,
Ptr, Chain, dl));
4894 DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl));
4901 DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl));
4907 EVT ModeVT =
Node->getValueType(0);
4909 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4910 SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr,
4911 Node->getOperand(0), dl);
4913 ModeVT, dl, Chain, StackPtr,
4923 EVT ModeVT =
Mode.getValueType();
4925 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4926 SDValue StInst = DAG.getStore(
4927 Node->getOperand(0), dl, Mode, StackPtr,
4930 DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl));
4938 EVT PtrTy = TLI.getPointerTy(
DL);
4940 Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
4941 Node->getOperand(0), dl));
4948 LLVM_DEBUG(
dbgs() <<
"Successfully converted node to libcall\n");
4957 MVT EltVT,
MVT NewEltVT) {
4959 MVT MidVT = OldEltsPerNewElt == 1
4966void SelectionDAGLegalize::PromoteNode(
SDNode *
Node) {
4969 MVT OVT =
Node->getSimpleValueType(0);
4975 OVT =
Node->getOperand(0).getSimpleValueType();
4981 OVT =
Node->getOperand(1).getSimpleValueType();
4984 OVT =
Node->getOperand(2).getSimpleValueType();
4985 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), OVT);
4987 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4988 switch (
Node->getOpcode()) {
5008 DAG.getConstant(TopBit, dl, NVT));
5012 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5026 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5029 DAG.getConstant(DiffBits, dl,
5030 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
5043 Results.push_back(PromoteLegalFP_TO_INT_SAT(
Node, dl));
5060 &&
"VAARG promotion is supported only for vectors or integer types");
5065 Tmp1 = DAG.getVAArg(NVT, dl, Chain,
Ptr,
Node->getOperand(2),
5066 Node->getConstantOperandVal(3));
5069 Tmp2 = DAG.
getNode(TruncOp, dl, OVT, Tmp1);
5073 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), Tmp2);
5074 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
5076 UpdatedNodes->insert(Tmp2.
getNode());
5077 UpdatedNodes->insert(Chain.
getNode());
5094 unsigned ExtOp, TruncOp;
5101 switch (
Node->getOpcode()) {
5117 if (TLI.isSExtCheaperThanZExt(OVT, NVT))
5126 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5127 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5129 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5130 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
5138 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5139 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5142 auto &
DL = DAG.getDataLayout();
5146 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(
DL, NVT)));
5152 unsigned ExtOp, TruncOp;
5153 if (
Node->getValueType(0).isVector() ||
5157 }
else if (
Node->getValueType(0).isInteger()) {
5164 Tmp1 =
Node->getOperand(0);
5166 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5167 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5169 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5172 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1);
5174 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1,
5175 DAG.getIntPtrConstant(0, dl));
5187 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5196 Node->getOperand(2));
5204 MVT CVT =
Node->getSimpleValueType(0);
5205 assert(CVT == OVT &&
"not handled");
5213 if (TLI.isCondCodeLegal(CCCode, CVT)) {
5214 Tmp1 =
Node->getOperand(0);
5215 Tmp2 =
Node->getOperand(1);
5217 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5218 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5221 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5222 Tmp4 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5232 DAG.getIntPtrConstant(0, dl,
true));
5244 TLI.isSExtCheaperThanZExt(
Node->getOperand(0).getValueType(), NVT))
5249 if (
Node->isStrictFPOpcode()) {
5251 std::tie(Tmp1, std::ignore) =
5252 DAG.getStrictFPExtendOrRound(
Node->getOperand(1), InChain, dl, NVT);
5253 std::tie(Tmp2, std::ignore) =
5254 DAG.getStrictFPExtendOrRound(
Node->getOperand(2), InChain, dl, NVT);
5256 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
5257 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
5258 Results.push_back(DAG.getNode(
Node->getOpcode(), dl, VTs,
5259 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5264 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5265 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5267 Tmp2,
Node->getOperand(2),
Node->getFlags()));
5274 cast<CondCodeSDNode>(
Node->getOperand(1))->get();
5277 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5278 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5280 Node->getOperand(0),
Node->getOperand(1),
5281 Tmp1, Tmp2,
Node->getOperand(4)));
5296 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
5300 DAG.getIntPtrConstant(0, dl,
true)));
5311 {
Node->getOperand(0),
Node->getOperand(1)});
5313 {
Node->getOperand(0),
Node->getOperand(2)});
5316 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5317 {Tmp3, Tmp1, Tmp2});
5319 {Tmp1.
getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5329 DAG.getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5330 DAG.getIntPtrConstant(0, dl,
true)));
5334 {
Node->getOperand(0),
Node->getOperand(1)});
5336 {
Node->getOperand(0),
Node->getOperand(2)});
5338 {
Node->getOperand(0),
Node->getOperand(3)});
5341 Tmp4 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5342 {Tmp4, Tmp1, Tmp2, Tmp3});
5344 {Tmp4.
getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5352 Tmp2 =
Node->getOperand(1);
5353 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5363 DAG.getIntPtrConstant(isTrunc, dl,
true)));
5368 {
Node->getOperand(0),
Node->getOperand(1)});
5369 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5372 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5382 DAG.getIntPtrConstant(0, dl,
true)));
5407 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5410 DAG.getIntPtrConstant(0, dl,
true)));
5428 {
Node->getOperand(0),
Node->getOperand(1)});
5429 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5432 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5447 "Invalid promote type for build_vector");
5453 for (
unsigned I = 0, E =
Node->getNumOperands();
I != E; ++
I) {
5482 "Invalid promote type for extract_vector_elt");
5489 EVT IdxVT =
Idx.getValueType();
5491 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
5497 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5498 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5506 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
5528 "Invalid promote type for insert_vector_elt");
5536 EVT IdxVT =
Idx.getValueType();
5539 SDValue Factor = DAG.getConstant(NewEltsPerOldElt,
SDLoc(), IdxVT);
5546 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5547 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5551 CastVal, IdxOffset);
5554 NewVec, Elt, InEltIdx);
5593 "unexpected promotion type");
5595 "unexpected atomic_swap with illegal type");
5599 DAG.getVTList(NVT, MVT::Other),
5600 { AM->getChain(), AM->getBasePtr(), CastVal },
5608 MVT ScalarType =
Scalar.getSimpleValueType();
5612 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5617 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5620 DAG.getIntPtrConstant(0, dl,
true)));
5646 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes);
5653 bool AnyLegalized =
false;
5664 if (LegalizedNodes.
insert(
N).second) {
5665 AnyLegalized =
true;
5686 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes, &UpdatedNodes);
5693 return LegalizedNodes.
count(
N);
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
static MaybeAlign getAlign(Value *Ptr)
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI)
Return true if sincos libcall is available.
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
static MachineMemOperand * getStackAlignedMMO(SDValue StackPtr, MachineFunction &MF, bool isObjectScalable)
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This is an SDNode representing atomic operations.
const SDValue & getVal() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
const BasicBlock & back() const
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOStore
The memory access writes data.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetLowering & getTargetLoweringInfo() const
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
allnodes_const_iterator allnodes_end() const
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const DataLayout & getDataLayout() const
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
LLVMContext * getContext() const
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
Information about stack frame layout on the target.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
constexpr ScalarTy getFixedValue() const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ FRAME_TO_ARGS_OFFSET
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
@ RESET_FPENV
Set floating-point environment to default state.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ GLOBAL_OFFSET_TABLE
The address of the GOT.
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ SET_FPENV_MEM
Sets the current floating point environment.
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
@ Undef
Value of the register doesn't matter.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
int32_t ExponentType
A signed type to represent a floating point numbers unbiased exponent.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)