59#define DEBUG_TYPE "legalizedag"
65struct FloatSignAsInt {
88class SelectionDAGLegalize {
100 EVT getSetCCResultType(
EVT VT)
const {
111 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
132 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC,
SDNode *
Node,
134 bool IsSigned,
EVT RetVT);
135 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC,
SDNode *
Node,
bool isSigned);
137 void ExpandFPLibCall(
SDNode *
Node, RTLIB::Libcall LC,
141 ExpandFastFPLibCall(
SDNode *
Node,
bool IsFast,
142 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
143 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
144 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
145 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
146 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
150 RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
151 RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
153 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
154 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
155 RTLIB::Libcall Call_PPCF128,
158 RTLIB::Libcall CallI64,
159 RTLIB::Libcall CallI128);
173 void getSignAsIntValue(FloatSignAsInt &State,
const SDLoc &
DL,
175 SDValue modifySignAsInt(
const FloatSignAsInt &State,
const SDLoc &
DL,
223 dbgs() <<
" with: "; New->dump(&DAG));
226 "Replacing one node with another that produces a different number "
230 UpdatedNodes->
insert(New);
236 dbgs() <<
" with: "; New->dump(&DAG));
240 UpdatedNodes->
insert(New.getNode());
241 ReplacedNode(Old.getNode());
248 for (
unsigned i = 0, e = Old->
getNumValues(); i != e; ++i) {
259 dbgs() <<
" with: "; New->dump(&DAG));
263 UpdatedNodes->
insert(New.getNode());
264 ReplacedNode(Old.getNode());
274 bool isObjectScalable) {
282 ObjectSize, MFI.getObjectAlign(FI));
289SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
294 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
296 assert(NumEltsGrowth &&
"Cannot promote to vector type with fewer elts!");
298 if (NumEltsGrowth == 1)
301 SmallVector<int, 8> NewMask;
302 for (
unsigned i = 0; i != NumMaskElts; ++i) {
304 for (
unsigned j = 0;
j != NumEltsGrowth; ++
j) {
308 NewMask.
push_back(Idx * NumEltsGrowth + j);
311 assert(NewMask.
size() == NumDestElts &&
"Non-integer NumEltsGrowth?");
317SDValue SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP) {
337 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
376SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
408 SmallVector<int, 8> ShufOps;
409 for (
unsigned i = 0; i != NumElts; ++i)
410 ShufOps.
push_back(i != InsertPos->getZExtValue() ? i : NumElts);
415 return ExpandInsertToVectorThroughStack(
Op);
418SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
433 AAMDNodes AAInfo =
ST->getAAInfo();
444 bitcastToAPInt().zextOrTrunc(32),
445 SDLoc(CFP), MVT::i32);
446 return DAG.
getStore(Chain, dl, Con, Ptr,
ST->getPointerInfo(),
447 ST->getBaseAlign(), MMOFlags, AAInfo);
455 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
456 return DAG.
getStore(Chain, dl, Con, Ptr,
ST->getPointerInfo(),
457 ST->getBaseAlign(), MMOFlags, AAInfo);
471 ST->getBaseAlign(), MMOFlags, AAInfo);
474 ST->getPointerInfo().getWithOffset(4),
475 ST->getBaseAlign(), MMOFlags, AAInfo);
484void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
491 MMOMetadata
Metadata =
ST->getMMOMetadataForSubAccess();
493 if (!
ST->isTruncatingStore()) {
495 if (SDNode *OptStore = OptimizeFloatStore(ST).
getNode()) {
496 ReplaceNode(ST, OptStore);
501 MVT VT =
Value.getSimpleValueType();
504 case TargetLowering::Legal: {
507 EVT MemVT =
ST->getMemoryVT();
510 *
ST->getMemOperand())) {
513 ReplaceNode(
SDValue(ST, 0), Result);
518 case TargetLowering::Custom: {
521 if (Res && Res !=
SDValue(Node, 0))
522 ReplaceNode(
SDValue(Node, 0), Res);
525 case TargetLowering::Promote: {
528 "Can only promote stores to same size type");
532 ReplaceNode(
SDValue(Node, 0), Result);
535 case TargetLowering::Expand:
545 EVT StVT =
ST->getMemoryVT();
550 if (StWidth != StSize) {
559 ReplaceNode(
SDValue(Node, 0), Result);
564 unsigned LogStWidth =
Log2_32(StWidthBits);
566 unsigned RoundWidth = 1 << LogStWidth;
567 assert(RoundWidth < StWidthBits);
568 unsigned ExtraWidth = StWidthBits - RoundWidth;
569 assert(ExtraWidth < RoundWidth);
570 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
571 "Store size not an integral number of bytes!");
575 unsigned IncrementSize;
577 if (
DL.isLittleEndian()) {
581 RoundVT,
ST->getBaseAlign(), MMOFlags,
Metadata);
584 IncrementSize = RoundWidth / 8;
591 ST->getPointerInfo().getWithOffset(IncrementSize),
592 ExtraVT,
ST->getBaseAlign(), MMOFlags,
Metadata);
604 IncrementSize = RoundWidth / 8;
609 ST->getPointerInfo().getWithOffset(IncrementSize),
610 ExtraVT,
ST->getBaseAlign(), MMOFlags,
Metadata);
615 ReplaceNode(
SDValue(Node, 0), Result);
618 ST->getAlign(),
ST->getAddressSpace())) {
621 case TargetLowering::Legal: {
622 EVT MemVT =
ST->getMemoryVT();
626 *
ST->getMemOperand())) {
628 ReplaceNode(
SDValue(ST, 0), Result);
632 case TargetLowering::Custom: {
634 if (Res && Res !=
SDValue(Node, 0))
635 ReplaceNode(
SDValue(Node, 0), Res);
638 case TargetLowering::Expand:
640 "Vector Stores are handled in LegalizeVectorOps");
660 ReplaceNode(
SDValue(Node, 0), Result);
666void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
675 LLVM_DEBUG(
dbgs() <<
"Legalizing non-extending load operation\n");
676 MVT VT =
Node->getSimpleValueType(0);
682 case TargetLowering::Legal: {
683 EVT MemVT =
LD->getMemoryVT();
688 *
LD->getMemOperand())) {
693 case TargetLowering::Custom:
700 case TargetLowering::Promote: {
703 "Can only promote loads to same size type");
707 if (
const MDNode *MD =
LD->getRanges()) {
711 LD->getMemOperand()->clearRanges();
718 case TargetLowering::Expand:
723 if (RChain.
getNode() != Node) {
724 assert(RVal.
getNode() != Node &&
"Load must be completely replaced");
728 UpdatedNodes->insert(RVal.
getNode());
729 UpdatedNodes->insert(RChain.
getNode());
737 EVT SrcVT =
LD->getMemoryVT();
740 MMOMetadata
Metadata =
LD->getMMOMetadataForSubAccess();
752 LD->getAddressSpace(), ExtType,
753 false) == TargetLowering::Promote)) {
767 Chain, Ptr,
LD->getPointerInfo(), NVT,
780 Result.getValueType(), Result,
789 unsigned LogSrcWidth =
Log2_32(SrcWidthBits);
791 unsigned RoundWidth = 1 << LogSrcWidth;
792 assert(RoundWidth < SrcWidthBits);
793 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
794 assert(ExtraWidth < RoundWidth);
795 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
796 "Load size not an integral number of bytes!");
800 unsigned IncrementSize;
803 if (
DL.isLittleEndian()) {
807 LD->getPointerInfo(), RoundVT,
LD->getBaseAlign(),
811 IncrementSize = RoundWidth / 8;
815 LD->getPointerInfo().getWithOffset(IncrementSize),
816 ExtraVT,
LD->getBaseAlign(), MMOFlags,
Metadata);
835 LD->getPointerInfo(), RoundVT,
LD->getBaseAlign(),
839 IncrementSize = RoundWidth / 8;
843 LD->getPointerInfo().getWithOffset(IncrementSize),
844 ExtraVT,
LD->getBaseAlign(), MMOFlags,
Metadata);
862 bool isCustom =
false;
864 LD->getAlign(),
LD->getAddressSpace(), ExtType,
868 case TargetLowering::Custom:
871 case TargetLowering::Legal:
883 EVT MemVT =
LD->getMemoryVT();
886 *
LD->getMemOperand())) {
892 case TargetLowering::Expand: {
893 EVT DestVT =
Node->getValueType(0);
903 LD->getAddressSpace(), ExtType,
false))) {
910 SrcVT,
LD->getMemOperand());
923 if (SVT == MVT::f16 || SVT == MVT::bf16) {
930 Ptr, ISrcVT,
LD->getMemOperand());
934 Chain =
Result.getValue(1);
940 "Vector Loads are handled in LegalizeVectorOps");
947 "EXTLOAD should always be supported!");
951 Node->getValueType(0),
953 LD->getMemOperand());
962 Chain =
Result.getValue(1);
971 assert(
Value.getNode() != Node &&
"Load must be completely replaced");
975 UpdatedNodes->insert(
Value.getNode());
976 UpdatedNodes->insert(Chain.
getNode());
983void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
992 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
994 TargetLowering::TypeLegal &&
995 "Unexpected illegal type!");
999 TargetLowering::TypeLegal ||
1002 "Unexpected illegal type!");
1006 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1007 bool SimpleFinishLegalizing =
true;
1008 switch (
Node->getOpcode()) {
1018 Node->getValueType(0));
1022 Node->getValueType(0));
1023 if (Action != TargetLowering::Promote)
1029 Node->getOperand(1).getValueType());
1041 Node->getOperand(0).getValueType());
1055 Node->getOperand(1).getValueType());
1064 Node->getOperand(1).getValueType());
1088 unsigned Opc =
Node->getOpcode();
1099 MVT OpVT =
Node->getOperand(CompareOperand).getSimpleValueType();
1102 if (Action == TargetLowering::Legal) {
1105 Node->getValueType(0));
1115 SimpleFinishLegalizing =
false;
1122 SimpleFinishLegalizing =
false;
1136 if (Action == TargetLowering::Legal)
1137 Action = TargetLowering::Expand;
1148 if (Action == TargetLowering::Legal)
1149 Action = TargetLowering::Custom;
1167 Action = TargetLowering::Legal;
1171 if (Action == TargetLowering::Expand) {
1175 Node->getOperand(0));
1176 ReplaceNode(Node, NewVal.
getNode());
1183 if (Action == TargetLowering::Expand) {
1187 Node->getOperand(0));
1188 ReplaceNode(Node, NewVal.
getNode());
1213 unsigned Scale =
Node->getConstantOperandVal(2);
1215 Node->getValueType(0), Scale);
1226 case ISD::VP_SCATTER:
1236 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1260 Node->getOpcode(),
Node->getOperand(0).getValueType());
1264 case ISD::VP_REDUCE_FADD:
1265 case ISD::VP_REDUCE_FMUL:
1266 case ISD::VP_REDUCE_ADD:
1267 case ISD::VP_REDUCE_MUL:
1268 case ISD::VP_REDUCE_AND:
1269 case ISD::VP_REDUCE_OR:
1270 case ISD::VP_REDUCE_XOR:
1271 case ISD::VP_REDUCE_SMAX:
1272 case ISD::VP_REDUCE_SMIN:
1273 case ISD::VP_REDUCE_UMAX:
1274 case ISD::VP_REDUCE_UMIN:
1275 case ISD::VP_REDUCE_FMAX:
1276 case ISD::VP_REDUCE_FMIN:
1277 case ISD::VP_REDUCE_FMAXIMUM:
1278 case ISD::VP_REDUCE_FMINIMUM:
1279 case ISD::VP_REDUCE_SEQ_FADD:
1280 case ISD::VP_REDUCE_SEQ_FMUL:
1282 Node->getOpcode(),
Node->getOperand(1).getValueType());
1286 case ISD::VP_CTTZ_ELTS:
1287 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
1289 Node->getOperand(0).getValueType());
1294 Node->getOpcode(),
Node->getNumOperands(),
Node->getValueType(0));
1310 if (SimpleFinishLegalizing) {
1311 SDNode *NewNode =
Node;
1312 switch (
Node->getOpcode()) {
1359 if (NewNode != Node) {
1360 ReplaceNode(Node, NewNode);
1364 case TargetLowering::Legal:
1367 case TargetLowering::Custom:
1375 if (
Node->getNumValues() == 1) {
1379 Node->getValueType(0) == MVT::Glue) &&
1380 "Type mismatch for custom legalized operation");
1383 ReplaceNode(
SDValue(Node, 0), Res);
1388 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i) {
1392 Node->getValueType(i) == MVT::Glue) &&
1393 "Type mismatch for custom legalized operation");
1397 ReplaceNode(Node, ResultVals.
data());
1402 case TargetLowering::Expand:
1403 if (ExpandNode(Node))
1406 case TargetLowering::LibCall:
1407 ConvertNodeToLibcall(Node);
1409 case TargetLowering::Promote:
1415 switch (
Node->getOpcode()) {
1428 return LegalizeLoadOps(Node);
1430 return LegalizeStoreOps(Node);
1434SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(
SDValue Op) {
1447 SmallPtrSet<const SDNode *, 32> Visited;
1454 if (
ST->isIndexed() ||
ST->isTruncatingStore() ||
1455 ST->getValue() != Vec)
1460 if (!
ST->getChain().reachesChainWithoutSideEffects(DAG.
getEntryNode()))
1469 ST->hasPredecessor(
Op.getNode()))
1489 Align ElementAlignment =
1494 if (
Op.getValueType().isVector()) {
1496 Op.getValueType(), Idx);
1497 NewLoad = DAG.
getLoad(
Op.getValueType(), dl, Ch, StackPtr,
1498 MachinePointerInfo(), ElementAlignment);
1512 NewLoadOperands[0] = Ch;
1518SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(
SDValue Op) {
1519 assert(
Op.getValueType().isVector() &&
"Non-vector insert subvector!");
1531 MachinePointerInfo PtrInfo =
1535 Align BaseVecAlignment =
1553 Ch, dl, Part, SubStackPtr,
1562 Ch, dl, Part, SubStackPtr,
1568 "ElementAlignment does not match!");
1571 return DAG.
getLoad(
Op.getValueType(), dl, Ch, StackPtr, PtrInfo,
1575SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
1579 unsigned NumOperands =
Node->getNumOperands();
1581 EVT VectorValueType =
Node->getOperand(0).getValueType();
1585 for (
unsigned I = 0;
I < NumOperands; ++
I) {
1587 for (
unsigned Idx = 0;
Idx < NumSubElem; ++
Idx) {
1596SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1599 "Unexpected opcode!");
1605 EVT VT =
Node->getValueType(0);
1607 :
Node->getOperand(0).getValueType();
1611 MachinePointerInfo PtrInfo =
1617 assert(TypeByteSize > 0 &&
"Vector element type too small for stack store!");
1622 MemVT.
bitsLT(
Node->getOperand(0).getValueType());
1625 for (
unsigned i = 0, e =
Node->getNumOperands(); i != e; ++i) {
1627 if (
Node->getOperand(i).isUndef())
continue;
1629 unsigned Offset = TypeByteSize*i;
1636 Node->getOperand(i), Idx,
1644 if (!Stores.
empty())
1650 return DAG.
getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1656void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1659 EVT FloatVT =
Value.getValueType();
1661 State.FloatVT = FloatVT;
1667 State.SignBit = NumBits - 1;
1682 State.FloatPointerInfo);
1685 if (DataLayout.isBigEndian()) {
1689 State.IntPointerInfo = State.FloatPointerInfo;
1692 unsigned ByteOffset = (NumBits / 8) - 1;
1699 State.IntPtr = IntPtr;
1701 State.IntPointerInfo, MVT::i8);
1708SDValue SelectionDAGLegalize::modifySignAsInt(
const FloatSignAsInt &State,
1716 State.IntPointerInfo, MVT::i8);
1717 return DAG.
getLoad(State.FloatVT,
DL, Chain, State.FloatPtr,
1718 State.FloatPointerInfo);
1721SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node)
const {
1726 if (
Sign.getValueType().isVector())
1730 FloatSignAsInt SignAsInt;
1731 getSignAsIntValue(SignAsInt,
DL,
Sign);
1751 FloatSignAsInt MagAsInt;
1752 getSignAsIntValue(MagAsInt,
DL, Mag);
1759 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1760 EVT ShiftVT = IntVT;
1766 if (ShiftAmount > 0) {
1769 }
else if (ShiftAmount < 0) {
1782 return modifySignAsInt(MagAsInt,
DL, CopiedSign);
1785SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node)
const {
1788 if (
Node->getValueType(0).isVector())
1791 FloatSignAsInt SignAsInt;
1792 getSignAsIntValue(SignAsInt,
DL,
Node->getOperand(0));
1801 return modifySignAsInt(SignAsInt,
DL, SignFlip);
1804SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node)
const {
1809 EVT FloatVT =
Value.getValueType();
1819 FloatSignAsInt ValueAsInt;
1820 getSignAsIntValue(ValueAsInt,
DL,
Value);
1825 return modifySignAsInt(ValueAsInt,
DL, ClearedSign);
1828void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1829 SmallVectorImpl<SDValue> &
Results) {
1831 assert(
SPReg &&
"Target cannot require DYNAMIC_STACKALLOC expansion and"
1832 " not tell us which reg is the stack pointer!");
1834 EVT VT =
Node->getValueType(0);
1846 Chain =
SP.getValue(1);
1855 if (Alignment > StackAlign)
1866SDValue SelectionDAGLegalize::EmitStackConvert(
SDValue SrcOp, EVT SlotVT,
1867 EVT DestVT,
const SDLoc &dl) {
1868 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.
getEntryNode());
1871SDValue SelectionDAGLegalize::EmitStackConvert(
SDValue SrcOp, EVT SlotVT,
1872 EVT DestVT,
const SDLoc &dl,
1879 SrcVT, SlotVT, DestAlign,
1881 (SlotVT.
bitsLT(DestVT) &&
1890SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1902 Node->getValueType(0).getVectorElementType());
1904 Node->getValueType(0), dl, Ch, StackPtr,
1911 unsigned NumElems =
Node->getNumOperands();
1913 EVT VT =
Node->getValueType(0);
1925 for (
unsigned i = 0; i < NumElems; ++i) {
1936 while (IntermedVals.
size() > 2) {
1937 NewIntermedVals.
clear();
1938 for (
unsigned i = 0, e = (IntermedVals.
size() & ~1u); i < e; i += 2) {
1944 FinalIndices.
reserve(IntermedVals[i].second.
size() +
1945 IntermedVals[i+1].second.
size());
1948 for (
unsigned j = 0, f = IntermedVals[i].second.
size(); j != f;
1951 FinalIndices.
push_back(IntermedVals[i].second[j]);
1953 for (
unsigned j = 0, f = IntermedVals[i+1].second.
size(); j != f;
1955 ShuffleVec[k] = NumElems + j;
1956 FinalIndices.
push_back(IntermedVals[i+1].second[j]);
1962 IntermedVals[i+1].first,
1964 else if (!
TLI.isShuffleMaskLegal(ShuffleVec, VT))
1967 std::make_pair(Shuffle, std::move(FinalIndices)));
1972 if ((IntermedVals.
size() & 1) != 0)
1975 IntermedVals.
swap(NewIntermedVals);
1979 "Invalid number of intermediate vectors");
1980 SDValue Vec1 = IntermedVals[0].first;
1982 if (IntermedVals.
size() > 1)
1983 Vec2 = IntermedVals[1].first;
1988 for (
unsigned i = 0, e = IntermedVals[0].second.
size(); i != e; ++i)
1989 ShuffleVec[IntermedVals[0].second[i]] = i;
1990 for (
unsigned i = 0, e = IntermedVals[1].second.
size(); i != e; ++i)
1991 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1995 else if (!
TLI.isShuffleMaskLegal(ShuffleVec, VT))
2004SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
2005 unsigned NumElems =
Node->getNumOperands();
2008 EVT VT =
Node->getValueType(0);
2009 EVT OpVT =
Node->getOperand(0).getValueType();
2014 bool isOnlyLowElement =
true;
2015 bool MoreThanTwoValues =
false;
2017 for (
unsigned i = 0; i < NumElems; ++i) {
2022 isOnlyLowElement =
false;
2028 }
else if (!Value2.
getNode()) {
2031 }
else if (
V != Value1 &&
V != Value2) {
2032 MoreThanTwoValues =
true;
2039 if (isOnlyLowElement)
2045 for (
unsigned i = 0, e = NumElems; i !=
e; ++i) {
2046 if (ConstantFPSDNode *
V =
2049 }
else if (ConstantSDNode *
V =
2052 CV.
push_back(
const_cast<ConstantInt *
>(
V->getConstantIntValue()));
2057 const ConstantInt *CI =
V->getConstantIntValue();
2078 SmallSet<SDValue, 16> DefinedValues;
2079 for (
unsigned i = 0; i < NumElems; ++i) {
2080 if (
Node->getOperand(i).isUndef())
2086 if (!MoreThanTwoValues) {
2087 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2088 for (
unsigned i = 0; i < NumElems; ++i) {
2092 ShuffleVec[i] =
V == Value1 ? 0 : NumElems;
2114 return ExpandVectorBuildThroughStack(Node);
2117SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2119 EVT VT =
Node->getValueType(0);
2130std::pair<SDValue, SDValue>
2131SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2132 TargetLowering::ArgListTy &&Args,
2133 bool IsSigned, EVT RetVT) {
2137 if (LCImpl != RTLIB::Unsupported)
2142 Node->getOperationName(&DAG));
2159 (RetTy ==
F.getReturnType() ||
F.getReturnType()->
isVoidTy()) &&
2166 TargetLowering::CallLoweringInfo CLI(DAG);
2168 CLI.setDebugLoc(SDLoc(Node))
2171 Callee, std::move(Args))
2172 .setTailCall(isTailCall)
2173 .setSExtResult(signExtend)
2174 .setZExtResult(!signExtend)
2175 .setIsPostTypeLegalization(
true);
2179 if (!CallInfo.second.getNode()) {
2185 LLVM_DEBUG(
dbgs() <<
"Created libcall: "; CallInfo.first.dump(&DAG));
2189std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2191 TargetLowering::ArgListTy
Args;
2193 EVT ArgVT =
Op.getValueType();
2195 TargetLowering::ArgListEntry
Entry(
Op, ArgTy);
2198 Args.push_back(Entry);
2201 return ExpandLibCall(LC, Node, std::move(Args),
isSigned,
2202 Node->getValueType(0));
2205void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2207 SmallVectorImpl<SDValue> &
Results) {
2208 if (LC == RTLIB::UNKNOWN_LIBCALL)
2211 if (
Node->isStrictFPOpcode()) {
2212 EVT RetVT =
Node->getValueType(0);
2214 if (LCImpl == RTLIB::Unsupported) {
2216 Node->getOperationName(&DAG));
2222 TargetLowering::MakeLibCallOptions CallOptions;
2226 DAG, LCImpl, RetVT,
Ops, CallOptions, SDLoc(Node),
Node->getOperand(0));
2228 Results.push_back(Tmp.second);
2231 SDValue Tmp = ExpandLibCall(LC, Node, IsSignedArgument).first;
2237void SelectionDAGLegalize::ExpandFastFPLibCall(
2238 SDNode *Node,
bool IsFast,
2239 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
2240 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
2241 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
2242 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
2243 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
2244 SmallVectorImpl<SDValue> &
Results) {
2246 EVT VT =
Node->getSimpleValueType(0);
2255 Call_F128.first, Call_PPCF128.first);
2261 Call_F80.second, Call_F128.second,
2262 Call_PPCF128.second);
2265 ExpandFPLibCall(Node, LC,
Results);
2268SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node,
bool isSigned,
2269 RTLIB::Libcall Call_I8,
2270 RTLIB::Libcall Call_I16,
2271 RTLIB::Libcall Call_I32,
2272 RTLIB::Libcall Call_I64,
2273 RTLIB::Libcall Call_I128) {
2275 switch (
Node->getSimpleValueType(0).SimpleTy) {
2277 case MVT::i8: LC = Call_I8;
break;
2278 case MVT::i16: LC = Call_I16;
break;
2279 case MVT::i32: LC = Call_I32;
break;
2280 case MVT::i64: LC = Call_I64;
break;
2281 case MVT::i128: LC = Call_I128;
break;
2283 return ExpandLibCall(LC, Node,
isSigned).first;
2288void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2289 RTLIB::Libcall Call_F32,
2290 RTLIB::Libcall Call_F64,
2291 RTLIB::Libcall Call_F80,
2292 RTLIB::Libcall Call_F128,
2293 RTLIB::Libcall Call_PPCF128,
2294 SmallVectorImpl<SDValue> &
Results) {
2295 EVT InVT =
Node->getOperand(
Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2297 Call_F32, Call_F64, Call_F80,
2298 Call_F128, Call_PPCF128);
2299 ExpandFPLibCall(Node, LC,
Results);
2302SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
2303 SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,
2304 RTLIB::Libcall CallI128) {
2306 switch (
Node->getSimpleValueType(0).SimpleTy) {
2327 EVT ArgVT =
Op.getValueType();
2329 TargetLowering::ArgListEntry Arg(
Op, ArgTy);
2331 Arg.IsZExt = !Arg.IsSExt;
2333 SDValue Res = ExpandLibCall(LC, Node, TargetLowering::ArgListTy{Arg},
2346SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2347 SmallVectorImpl<SDValue> &
Results) {
2348 unsigned Opcode =
Node->getOpcode();
2352 switch (
Node->getSimpleValueType(0).SimpleTy) {
2354 case MVT::i8: LC=
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
2355 case MVT::i16: LC=
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
2356 case MVT::i32: LC=
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
2357 case MVT::i64: LC=
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
2358 case MVT::i128: LC=
isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
2366 EVT RetVT =
Node->getValueType(0);
2369 TargetLowering::ArgListTy
Args;
2371 EVT ArgVT =
Op.getValueType();
2373 TargetLowering::ArgListEntry
Entry(
Op, ArgTy);
2376 Args.push_back(Entry);
2381 TargetLowering::ArgListEntry
Entry(
2382 FIPtr, PointerType::getUnqual(RetTy->
getContext()));
2385 Args.push_back(Entry);
2388 if (LibcallImpl == RTLIB::Unsupported) {
2390 Node->getOperationName(&DAG));
2401 TargetLowering::CallLoweringInfo CLI(DAG);
2405 RetTy, Callee, std::move(Args))
2413 MachinePointerInfo PtrInfo =
2416 SDValue Rem = DAG.
getLoad(RetVT, dl, CallInfo.second, FIPtr, PtrInfo);
2417 Results.push_back(CallInfo.first);
2425 return Libcalls.
getLibcallImpl(RTLIB::getSINCOS(VT)) != RTLIB::Unsupported ||
2446SDValue SelectionDAGLegalize::ExpandSincosStretLibCall(SDNode *Node)
const {
2452 RTLIB::Libcall LC = RTLIB::getSINCOS_STRET(ArgVT);
2454 if (SincosStret == RTLIB::Unsupported)
2462 const RTLIB::RuntimeLibcallsInfo &CallsInfo =
2470 Type *SincosStretRetTy = FuncTy->getReturnType();
2476 TargetLowering::ArgListTy
Args;
2480 if (FuncTy->getParamType(0)->isPointerTy()) {
2484 AttributeSet PtrAttrs = FuncAttrs.getParamAttrs(0);
2486 const uint64_t ByteSize =
DL.getTypeAllocSize(StructTy);
2487 const Align StackAlign =
DL.getPrefTypeAlign(StructTy);
2492 TargetLowering::ArgListEntry
Entry(SRet, FuncTy->getParamType(0));
2493 Entry.IsSRet =
true;
2494 Entry.IndirectType = StructTy;
2495 Entry.Alignment = StackAlign;
2497 Args.push_back(Entry);
2498 Args.emplace_back(Arg, FuncTy->getParamType(1));
2500 Args.emplace_back(Arg, FuncTy->getParamType(0));
2503 TargetLowering::CallLoweringInfo CLI(DAG);
2506 .setLibCallee(CallConv, SincosStretRetTy, Callee, std::move(Args))
2507 .setIsPostTypeLegalization();
2512 MachinePointerInfo PtrInfo =
2514 SDValue LoadSin = DAG.
getLoad(ArgVT, dl, CallResult.second, SRet, PtrInfo);
2523 SDVTList Tys = DAG.
getVTList(ArgVT, ArgVT);
2528 if (!CallResult.first.getValueType().isVector())
2529 return CallResult.first;
2537 SDVTList Tys = DAG.
getVTList(ArgVT, ArgVT);
2541SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node)
const {
2543 EVT VT =
Node->getValueType(0);
2546 EVT ExpVT =
N.getValueType();
2548 if (AsIntVT == EVT())
2562 SDNodeFlags NUW_NSW;
2570 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2571 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2572 const int Precision = APFloat::semanticsPrecision(FltSem);
2579 const APFloat One(FltSem,
"1.0");
2580 APFloat ScaleUpK =
scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2584 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2654 ExponentShiftAmt, NUW_NSW);
2659SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node)
const {
2663 EVT ExpVT =
Node->getValueType(1);
2665 if (AsIntVT == EVT())
2675 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2676 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2709 FractSignMaskVal.
setBit(BitSize - 1);
2716 const APFloat One(FltSem,
"1.0");
2720 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2732 SDValue AddNegSmallestNormal =
2734 SDValue DenormOrZero = DAG.
getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2767 const APFloat Half(FltSem,
"0.5");
2788SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2792 EVT DestVT =
Node->getValueType(0);
2794 unsigned OpNo =
Node->isStrictFPOpcode() ? 1 : 0;
2801 (DestVT.
bitsLE(MVT::f64) ||
2805 LLVM_DEBUG(
dbgs() <<
"32-bit [signed|unsigned] integer to float/double "
2829 MachinePointerInfo());
2834 DAG.
getStore(MemChain, dl,
Hi, HiPtr, MachinePointerInfo());
2839 DAG.
getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2848 if (
Node->isStrictFPOpcode()) {
2850 {
Node->getOperand(0),
Load, Bias});
2852 if (DestVT !=
Sub.getValueType()) {
2853 std::pair<SDValue, SDValue> ResultPair;
2856 Result = ResultPair.first;
2857 Chain = ResultPair.second;
2872 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2873 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2874 LLVM_DEBUG(
dbgs() <<
"Converting unsigned i32/i64 to f32/f64\n");
2889 EVT SetCCVT = getSetCCResultType(SrcVT);
2901 if (
Node->isStrictFPOpcode()) {
2909 Flags.setNoFPExcept(
Node->getFlags().hasNoFPExcept());
2912 Flags.setNoFPExcept(
true);
2935 "Cannot perform lossless SINT_TO_FP!");
2938 if (
Node->isStrictFPOpcode()) {
2940 {
Node->getOperand(0), Op0 });
2949 SignSet, Four,
Zero);
2958 case MVT::i8 : FF = 0x43800000ULL;
break;
2959 case MVT::i16: FF = 0x47800000ULL;
break;
2960 case MVT::i32: FF = 0x4F800000ULL;
break;
2961 case MVT::i64: FF = 0x5F800000ULL;
break;
2965 Constant *FudgeFactor = ConstantInt::get(
2974 if (DestVT == MVT::f32)
2984 HandleSDNode Handle(
Load);
2985 LegalizeOp(
Load.getNode());
2989 if (
Node->isStrictFPOpcode()) {
2991 { Tmp1.
getValue(1), Tmp1, FudgeInReg });
2992 Chain =
Result.getValue(1);
3004void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
3005 SDNode *
N,
const SDLoc &dl, SmallVectorImpl<SDValue> &
Results) {
3009 EVT DestVT =
N->getValueType(0);
3010 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
3017 unsigned OpToUse = 0;
3045 DAG.
getNode(OpToUse, dl, {DestVT, MVT::Other},
3048 dl, NewInTy, LegalOp)});
3055 DAG.
getNode(OpToUse, dl, DestVT,
3057 dl, NewInTy, LegalOp)));
3065void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *
N,
const SDLoc &dl,
3066 SmallVectorImpl<SDValue> &
Results) {
3067 bool IsStrict =
N->isStrictFPOpcode();
3070 EVT DestVT =
N->getValueType(0);
3073 EVT NewOutTy = DestVT;
3075 unsigned OpToUse = 0;
3099 SDVTList VTs = DAG.
getVTList(NewOutTy, MVT::Other);
3115SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
3117 unsigned Opcode =
Node->getOpcode();
3120 EVT NewOutTy =
Node->getValueType(0);
3132 Node->getOperand(1));
3138 EVT VT =
Op.getValueType();
3158SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
3160 MVT VecVT = IsVPOpcode ?
Node->getOperand(1).getSimpleValueType()
3161 :
Node->getOperand(0).getSimpleValueType();
3163 MVT ScalarVT =
Node->getSimpleValueType(0);
3170 assert(
Node->getOperand(0).getValueType().isFloatingPoint() &&
3171 "Only FP promotion is supported");
3173 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j)
3174 if (
Node->getOperand(j).getValueType().isVector() &&
3179 assert(
Node->getOperand(j).getValueType().isFloatingPoint() &&
3180 "Only FP promotion is supported");
3183 }
else if (
Node->getOperand(j).getValueType().isFloatingPoint()) {
3199bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3203 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3205 switch (
Node->getOpcode()) {
3250 Results.push_back(ExpandPARITY(
Node->getOperand(0), dl));
3301 SDVTList VTs = DAG.
getVTList(
Node->getValueType(0), MVT::Other);
3314 Node->getOperand(0),
Node->getOperand(2),
Node->getOperand(1),
3323 SDVTList VTs = DAG.
getVTList(
Node->getValueType(0), MVT::Other);
3326 Node->getOperand(0),
Node->getOperand(1),
Node->getOperand(2),
3334 EVT OuterType =
Node->getValueType(0);
3367 EVT VT =
Node->getValueType(0);
3372 RHS =
RHS->getOperand(0);
3376 Node->getOperand(0),
Node->getOperand(1),
3385 TargetLowering::Expand)
3389 EVT VT =
Node->getValueType(0);
3392 Node->getOperand(0),
Node->getOperand(1),
3400 EVT VT =
Node->getValueType(0);
3404 Node->getOperand(0),
Node->getOperand(1),
3411 ExpandDYNAMIC_STACKALLOC(Node,
Results);
3414 for (
unsigned i = 0; i <
Node->getNumValues(); i++)
3419 EVT VT =
Node->getValueType(0);
3436 Node->getValueType(0))
3437 == TargetLowering::Legal)
3441 if ((Tmp1 = EmitStackConvert(
Node->getOperand(1),
Node->getValueType(0),
3442 Node->getValueType(0), dl,
3443 Node->getOperand(0)))) {
3444 ReplaceNode(Node, Tmp1.
getNode());
3445 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_ROUND node\n");
3458 if ((Tmp1 = EmitStackConvert(
Node->getOperand(0),
Node->getValueType(0),
3459 Node->getValueType(0), dl)))
3470 Node->getValueType(0))
3471 == TargetLowering::Legal)
3475 if ((Tmp1 = EmitStackConvert(
3476 Node->getOperand(1),
Node->getOperand(1).getValueType(),
3477 Node->getValueType(0), dl,
Node->getOperand(0)))) {
3478 ReplaceNode(Node, Tmp1.
getNode());
3479 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_EXTEND node\n");
3485 EVT SrcVT =
Op.getValueType();
3486 EVT DstVT =
Node->getValueType(0);
3492 if ((Tmp1 = EmitStackConvert(
Op, SrcVT, DstVT, dl)))
3502 if (
Op.getValueType() == MVT::bf16) {
3512 if (
Node->getValueType(0) != MVT::f32)
3519 if (
Op.getValueType() != MVT::f32)
3531 if (
Node->getValueType(0) == MVT::bf16) {
3548 EVT DstVT =
Node->getValueType(0);
3563 EVT ResVT =
Node->getValueType(0);
3577 EVT VT =
Node->getValueType(0);
3609 if (
Node->isStrictFPOpcode())
3616 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
3618 if (
Node->isStrictFPOpcode())
3628 ReplaceNode(Node, Tmp1.
getNode());
3629 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_SINT node\n");
3642 ReplaceNodeWithValue(
SDValue(Node, 0), Tmp1);
3643 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_UINT node\n");
3655 EVT ResVT =
Node->getValueType(0);
3669 if (
Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3672 Node->getOperand(0));
3674 Tmp1 = ExpandExtractFromVectorThroughStack(
SDValue(Node, 0));
3678 Results.push_back(ExpandExtractFromVectorThroughStack(
SDValue(Node, 0)));
3681 Results.push_back(ExpandInsertToVectorThroughStack(
SDValue(Node, 0)));
3684 if (EVT VectorValueType =
Node->getOperand(0).getValueType();
3687 Results.push_back(ExpandVectorBuildThroughStack(Node));
3689 Results.push_back(ExpandConcatVectors(Node));
3692 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3701 EVT VT =
Node->getValueType(0);
3711 if (NewEltVT.
bitsLT(EltVT)) {
3727 unsigned int factor =
3735 for (
unsigned fi = 0; fi < factor; ++fi)
3739 for (
unsigned fi = 0; fi < factor; ++fi)
3750 for (
unsigned i = 0; i != NumElems; ++i) {
3783 EVT OpTy =
Node->getOperand(0).getValueType();
3784 if (
Node->getConstantOperandVal(1)) {
3793 Node->getOperand(0));
3804 Node->getValueType(0)));
3811 ?
"llvm.stackaddress"
3814 Twine(IntrinsicName) +
" is not supported on this target.",
3823 Node->getOperand(1)));
3833 Results.push_back(ExpandFCOPYSIGN(Node));
3836 Results.push_back(ExpandFNEG(Node));
3839 Results.push_back(ExpandFABS(Node));
3845 Test,
Node->getFlags(), SDLoc(Node), DAG))
3855 switch (
Node->getOpcode()) {
3862 Tmp1 =
Node->getOperand(0);
3863 Tmp2 =
Node->getOperand(1);
3887 EVT VT =
Node->getValueType(0);
3903 EVT VT =
Node->getValueType(0);
3904 RTLIB::Libcall LC = RTLIB::getLDEXP(VT);
3910 if (
SDValue Expanded = expandLdexp(Node)) {
3913 Results.push_back(Expanded.getValue(1));
3919 RTLIB::Libcall LC = RTLIB::getFREXP(
Node->getValueType(0));
3925 if (
SDValue Expanded = expandFrexp(Node)) {
3927 Results.push_back(Expanded.getValue(1));
3932 RTLIB::Libcall LC = RTLIB::getMODF(
Node->getValueType(0));
3944 EVT VT =
Node->getValueType(0);
3956 if (
Node->getValueType(0) != MVT::f32) {
3968 if (
Node->getValueType(0) != MVT::f32) {
3973 {Node->getOperand(0), Node->getOperand(1)});
3975 {
Node->getValueType(0), MVT::Other},
3985 MVT SVT =
Op.getSimpleValueType();
3986 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
4000 EVT VT =
Node->getValueType(0);
4001 TargetLowering::FPImmAction Action =
4004 case TargetLowering::FPImmAction::Legal:
4006 case TargetLowering::FPImmAction::Bitcast: {
4009 "integer type for FP immediate bits must be legal");
4017 case TargetLowering::FPImmAction::ConstantPool:
4018 Results.push_back(ExpandConstantFP(CFP));
4026 Results.push_back(ExpandConstant(CP));
4030 EVT VT =
Node->getValueType(0);
4033 const SDNodeFlags
Flags =
Node->getFlags();
4041 EVT VT =
Node->getValueType(0);
4044 "Don't know how to expand this subtraction!");
4045 Tmp1 = DAG.
getNOT(dl,
Node->getOperand(1), VT);
4059 EVT VT =
Node->getValueType(0);
4062 Tmp1 = DAG.
getNode(DivRemOpc, dl, VTs,
Node->getOperand(0),
4063 Node->getOperand(1));
4070 unsigned ExpandOpcode =
4072 EVT VT =
Node->getValueType(0);
4075 Tmp1 = DAG.
getNode(ExpandOpcode, dl, VTs,
Node->getOperand(0),
4076 Node->getOperand(1));
4084 EVT VT =
LHS.getValueType();
4099 TargetLowering::MulExpansionKind::Always)) {
4100 for (
unsigned i = 0; i < 2; ++i) {
4118 EVT VT =
Node->getValueType(0);
4129 unsigned OpToUse = 0;
4130 if (HasSMUL_LOHI && !HasMULHS) {
4132 }
else if (HasUMUL_LOHI && !HasMULHU) {
4134 }
else if (HasSMUL_LOHI) {
4136 }
else if (HasUMUL_LOHI) {
4141 Node->getOperand(1)));
4152 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
4209 Node->getOperand(0),
4210 Node->getOperand(1),
4211 Node->getConstantOperandVal(2),
4234 EVT VT =
LHS.getValueType();
4238 EVT CarryType =
Node->getValueType(1);
4239 EVT SetCCType = getSetCCResultType(
Node->getValueType(0));
4303 Tmp1 =
Node->getOperand(0);
4304 Tmp2 =
Node->getOperand(1);
4305 Tmp3 =
Node->getOperand(2);
4326 unsigned EntrySize =
4362 Tmp1 =
Node->getOperand(0);
4363 Tmp2 =
Node->getOperand(1);
4369 Node->getOperand(2));
4381 Node->getOperand(2));
4392 unsigned Offset = IsStrict ? 1 : 0;
4398 NeedInvert, dl, Chain, IsSignaling);
4406 {Chain, Tmp1, Tmp2, Tmp3},
Node->getFlags());
4409 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl,
Node->getValueType(0), Tmp1,
4410 Tmp2, Tmp3,
Node->getFlags());
4429 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
4433 EVT VT =
Node->getValueType(0);
4444 Tmp1 =
Node->getOperand(0);
4445 Tmp2 =
Node->getOperand(1);
4446 Tmp3 =
Node->getOperand(2);
4447 Tmp4 =
Node->getOperand(3);
4448 EVT VT =
Node->getValueType(0);
4458 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4460 EVT CCVT = getSetCCResultType(CmpVT);
4468 bool Legalized =
false;
4486 Tmp1 = DAG.
getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC,
4494 NeedInvert, dl, Chain);
4496 assert(Legalized &&
"Can't legalize SELECT_CC with legal condition!");
4507 Tmp2, Tmp3, Tmp4,
CC,
Node->getFlags());
4512 Tmp2, Tmp3, Tmp4,
CC,
Node->getFlags());
4521 Tmp1 =
Node->getOperand(0);
4522 Tmp2 =
Node->getOperand(2);
4523 Tmp3 =
Node->getOperand(3);
4524 Tmp4 =
Node->getOperand(1);
4528 Tmp2, Tmp3, Tmp4, NeedInvert, dl, Chain);
4530 assert(Legalized &&
"Can't legalize BR_CC with legal condition!");
4535 assert(!NeedInvert &&
"Don't know how to invert BR_CC!");
4538 Tmp4, Tmp2, Tmp3,
Node->getOperand(4));
4543 Tmp2, Tmp3,
Node->getOperand(4));
4549 Results.push_back(ExpandBUILD_VECTOR(Node));
4552 Results.push_back(ExpandSPLAT_VECTOR(Node));
4558 EVT VT =
Node->getValueType(0);
4564 for (
unsigned Idx = 0;
Idx < NumElem;
Idx++) {
4598 case ISD::VP_CTTZ_ELTS:
4599 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
4611 EVT ResVT =
Node->getValueType(0);
4643 switch (
Node->getOpcode()) {
4646 Node->getValueType(0))
4647 == TargetLowering::Legal)
4658 EVT VT =
Node->getValueType(0);
4659 const SDNodeFlags
Flags =
Node->getFlags();
4662 {Node->getOperand(0), Node->getOperand(1), Neg},
4678 Node->getOperand(1).getValueType())
4679 == TargetLowering::Legal)
4692 ReplaceNode(Node,
Results.data());
4704 return Flags.hasApproximateFuncs() && Flags.hasNoNaNs() &&
4705 Flags.hasNoInfs() && Flags.hasNoSignedZeros();
4708void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4712 TargetLowering::MakeLibCallOptions CallOptions;
4715 unsigned Opc =
Node->getOpcode();
4720 TargetLowering::ArgListTy
Args;
4722 TargetLowering::CallLoweringInfo CLI(DAG);
4724 .setChain(
Node->getOperand(0))
4726 CallingConv::C, Type::getVoidTy(*DAG.
getContext()),
4733 Results.push_back(CallResult.second);
4754 RTLIB::AtomicLibcall LC =
4756 EVT RetVT =
Node->getValueType(0);
4761 Ops.push_back(
Node->getOperand(1));
4766 std::pair<SDValue, SDValue> Tmp =
4768 Node->getOperand(0));
4770 Results.push_back(Tmp.second);
4775 TargetLowering::ArgListTy
Args;
4776 TargetLowering::CallLoweringInfo CLI(DAG);
4778 .setChain(
Node->getOperand(0))
4779 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.
getContext()),
4785 Results.push_back(CallResult.second);
4793 DAG, RTLIB::CLEAR_CACHE, MVT::isVoid, {StartVal, EndVal}, CallOptions,
4794 SDLoc(Node), InputChain);
4795 Results.push_back(Tmp.second);
4800 ExpandFPLibCall(Node, RTLIB::getFMIN(
Node->getSimpleValueType(0)),
Results);
4807 ExpandFPLibCall(Node, RTLIB::getFMAX(
Node->getSimpleValueType(0)),
Results);
4810 ExpandFPLibCall(Node, RTLIB::getFMINIMUM_NUM(
Node->getSimpleValueType(0)),
4814 ExpandFPLibCall(Node, RTLIB::getFMAXIMUM_NUM(
Node->getSimpleValueType(0)),
4822 {RTLIB::FAST_SQRT_F32, RTLIB::SQRT_F32},
4823 {RTLIB::FAST_SQRT_F64, RTLIB::SQRT_F64},
4824 {RTLIB::FAST_SQRT_F80, RTLIB::SQRT_F80},
4825 {RTLIB::FAST_SQRT_F128, RTLIB::SQRT_F128},
4826 {RTLIB::FAST_SQRT_PPCF128, RTLIB::SQRT_PPCF128},
4831 ExpandFPLibCall(Node, RTLIB::getCBRT(
Node->getSimpleValueType(0)),
Results);
4835 ExpandFPLibCall(Node, RTLIB::getSIN(
Node->getSimpleValueType(0)),
Results);
4839 ExpandFPLibCall(Node, RTLIB::getCOS(
Node->getSimpleValueType(0)),
Results);
4843 ExpandFPLibCall(Node, RTLIB::getTAN(
Node->getSimpleValueType(0)),
Results);
4847 ExpandFPLibCall(Node, RTLIB::getASIN(
Node->getSimpleValueType(0)),
Results);
4851 ExpandFPLibCall(Node, RTLIB::getACOS(
Node->getSimpleValueType(0)),
Results);
4855 ExpandFPLibCall(Node, RTLIB::getATAN(
Node->getSimpleValueType(0)),
Results);
4859 ExpandFPLibCall(Node, RTLIB::getATAN2(
Node->getSimpleValueType(0)),
4864 ExpandFPLibCall(Node, RTLIB::getSINH(
Node->getSimpleValueType(0)),
Results);
4868 ExpandFPLibCall(Node, RTLIB::getCOSH(
Node->getSimpleValueType(0)),
Results);
4872 ExpandFPLibCall(Node, RTLIB::getTANH(
Node->getSimpleValueType(0)),
Results);
4876 EVT VT =
Node->getValueType(0);
4879 RTLIB::Libcall SincosStret = RTLIB::getSINCOS_STRET(VT);
4880 if (SincosStret != RTLIB::UNKNOWN_LIBCALL) {
4881 if (
SDValue Expanded = ExpandSincosStretLibCall(Node)) {
4883 Results.push_back(Expanded.getValue(1));
4890 ? RTLIB::getSINCOS(VT)
4891 : RTLIB::getSINCOSPI(VT);
4895 Node->getOperationName(&DAG));
4905 ExpandFPLibCall(Node, RTLIB::getLOG(
Node->getSimpleValueType(0)),
Results);
4909 ExpandFPLibCall(Node, RTLIB::getLOG2(
Node->getSimpleValueType(0)),
Results);
4913 ExpandFPLibCall(Node, RTLIB::getLOG10(
Node->getSimpleValueType(0)),
4918 ExpandFPLibCall(Node, RTLIB::getEXP(
Node->getSimpleValueType(0)),
Results);
4922 ExpandFPLibCall(Node, RTLIB::getEXP2(
Node->getSimpleValueType(0)),
Results);
4925 ExpandFPLibCall(Node, RTLIB::getEXP10(
Node->getSimpleValueType(0)),
4930 ExpandFPLibCall(Node, RTLIB::getTRUNC(
Node->getSimpleValueType(0)),
4935 ExpandFPLibCall(Node, RTLIB::getFLOOR(
Node->getSimpleValueType(0)),
4940 ExpandFPLibCall(Node, RTLIB::getCEIL(
Node->getSimpleValueType(0)),
Results);
4944 ExpandFPLibCall(Node, RTLIB::getRINT(
Node->getSimpleValueType(0)),
Results);
4948 ExpandFPLibCall(Node, RTLIB::getNEARBYINT(
Node->getSimpleValueType(0)),
4953 ExpandFPLibCall(Node, RTLIB::getROUND(
Node->getSimpleValueType(0)),
4958 ExpandFPLibCall(Node, RTLIB::getROUNDEVEN(
Node->getSimpleValueType(0)),
4963 ExpandFPLibCall(Node, RTLIB::getLDEXP(
Node->getSimpleValueType(0)),
4968 EVT VT =
Node->getValueType(0);
4969 RTLIB::Libcall LC =
Node->getOpcode() ==
ISD::FMODF ? RTLIB::getMODF(VT)
4970 : RTLIB::getFREXP(VT);
4975 Node->getOperationName(&DAG));
4976 for (
unsigned I = 0,
E =
Node->getNumValues();
I !=
E; ++
I)
4983 RTLIB::Libcall LC = RTLIB::getPOWI(
Node->getSimpleValueType(0));
4984 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
4987 if (
Node->isStrictFPOpcode()) {
4990 {
Node->getValueType(0),
Node->getValueType(1)},
4991 {
Node->getOperand(0),
Node->getOperand(2)});
4994 {
Node->getValueType(0),
Node->getValueType(1)},
5001 Node->getOperand(1));
5003 Node->getValueType(0),
5008 unsigned Offset =
Node->isStrictFPOpcode() ? 1 : 0;
5009 bool ExponentHasSizeOfInt =
5011 Node->getOperand(1 +
Offset).getValueType().getSizeInBits();
5012 if (!ExponentHasSizeOfInt) {
5019 ExpandFPLibCall(Node, LC,
Results);
5024 ExpandFPLibCall(Node, RTLIB::getPOW(
Node->getSimpleValueType(0)),
Results);
5028 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
5029 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
5031 RTLIB::LROUND_PPCF128,
Results);
5035 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
5036 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
5037 RTLIB::LLROUND_F128,
5038 RTLIB::LLROUND_PPCF128,
Results);
5042 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
5043 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
5045 RTLIB::LRINT_PPCF128,
Results);
5049 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
5050 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
5052 RTLIB::LLRINT_PPCF128,
Results);
5057 {RTLIB::FAST_DIV_F32, RTLIB::DIV_F32},
5058 {RTLIB::FAST_DIV_F64, RTLIB::DIV_F64},
5059 {RTLIB::FAST_DIV_F80, RTLIB::DIV_F80},
5060 {RTLIB::FAST_DIV_F128, RTLIB::DIV_F128},
5061 {RTLIB::FAST_DIV_PPCF128, RTLIB::DIV_PPCF128},
Results);
5066 ExpandFPLibCall(Node, RTLIB::getREM(
Node->getSimpleValueType(0)),
Results);
5070 ExpandFPLibCall(Node, RTLIB::getFMA(
Node->getSimpleValueType(0)),
Results);
5075 {RTLIB::FAST_ADD_F32, RTLIB::ADD_F32},
5076 {RTLIB::FAST_ADD_F64, RTLIB::ADD_F64},
5077 {RTLIB::FAST_ADD_F80, RTLIB::ADD_F80},
5078 {RTLIB::FAST_ADD_F128, RTLIB::ADD_F128},
5079 {RTLIB::FAST_ADD_PPCF128, RTLIB::ADD_PPCF128},
Results);
5085 {RTLIB::FAST_MUL_F32, RTLIB::MUL_F32},
5086 {RTLIB::FAST_MUL_F64, RTLIB::MUL_F64},
5087 {RTLIB::FAST_MUL_F80, RTLIB::MUL_F80},
5088 {RTLIB::FAST_MUL_F128, RTLIB::MUL_F128},
5089 {RTLIB::FAST_MUL_PPCF128, RTLIB::MUL_PPCF128},
Results);
5093 if (
Node->getValueType(0) == MVT::f32) {
5094 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node,
false).first);
5098 if (
Node->getValueType(0) == MVT::f32) {
5100 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32,
Node->getOperand(1),
5101 CallOptions, SDLoc(Node),
Node->getOperand(0));
5103 Results.push_back(Tmp.second);
5107 if (
Node->getValueType(0) == MVT::f32) {
5109 DAG, RTLIB::FPEXT_F16_F32, MVT::f32,
Node->getOperand(1), CallOptions,
5110 SDLoc(Node),
Node->getOperand(0));
5112 Results.push_back(Tmp.second);
5119 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16");
5120 Results.push_back(ExpandLibCall(LC, Node,
false).first);
5126 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_bf16");
5127 Results.push_back(ExpandLibCall(LC, Node,
false).first);
5135 bool IsStrict =
Node->isStrictFPOpcode();
5138 EVT SVT =
Node->getOperand(IsStrict ? 1 : 0).getValueType();
5139 EVT RVT =
Node->getValueType(0);
5146 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5147 for (
unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
5148 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5156 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
5161 NVT,
Node->getOperand(IsStrict ? 1 : 0));
5163 std::pair<SDValue, SDValue> Tmp =
5167 Results.push_back(Tmp.second);
5175 bool IsStrict =
Node->isStrictFPOpcode();
5180 EVT SVT =
Op.getValueType();
5181 EVT RVT =
Node->getValueType(0);
5188 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5189 for (
unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
5190 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5198 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
5201 std::pair<SDValue, SDValue> Tmp =
5207 Results.push_back(Tmp.second);
5218 bool IsStrict =
Node->isStrictFPOpcode();
5221 EVT VT =
Node->getValueType(0);
5223 "Unable to expand as libcall if it is not normal rounding");
5226 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
5228 std::pair<SDValue, SDValue> Tmp =
5232 Results.push_back(Tmp.second);
5238 Node->getValueType(0)),
5239 Node,
false).first);
5245 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5252 Node->getValueType(0));
5254 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
5256 std::pair<SDValue, SDValue> Tmp =
5258 CallOptions, SDLoc(Node),
Node->getOperand(0));
5260 Results.push_back(Tmp.second);
5266 {RTLIB::FAST_SUB_F32, RTLIB::SUB_F32},
5267 {RTLIB::FAST_SUB_F64, RTLIB::SUB_F64},
5268 {RTLIB::FAST_SUB_F80, RTLIB::SUB_F80},
5269 {RTLIB::FAST_SUB_F128, RTLIB::SUB_F128},
5270 {RTLIB::FAST_SUB_PPCF128, RTLIB::SUB_PPCF128},
Results);
5274 Results.push_back(ExpandIntLibCall(Node,
true,
5276 RTLIB::SREM_I16, RTLIB::SREM_I32,
5277 RTLIB::SREM_I64, RTLIB::SREM_I128));
5280 Results.push_back(ExpandIntLibCall(Node,
false,
5282 RTLIB::UREM_I16, RTLIB::UREM_I32,
5283 RTLIB::UREM_I64, RTLIB::UREM_I128));
5286 Results.push_back(ExpandIntLibCall(Node,
true,
5288 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
5289 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
5292 Results.push_back(ExpandIntLibCall(Node,
false,
5294 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
5295 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
5300 ExpandDivRemLibCall(Node,
Results);
5303 Results.push_back(ExpandIntLibCall(Node,
false,
5305 RTLIB::MUL_I16, RTLIB::MUL_I32,
5306 RTLIB::MUL_I64, RTLIB::MUL_I128));
5309 Results.push_back(ExpandBitCountingLibCall(
5310 Node, RTLIB::CTLZ_I32, RTLIB::CTLZ_I64, RTLIB::CTLZ_I128));
5313 Results.push_back(ExpandBitCountingLibCall(
5314 Node, RTLIB::CTPOP_I32, RTLIB::CTPOP_I64, RTLIB::CTPOP_I128));
5343 EVT ModeVT =
Node->getValueType(0);
5347 Node->getOperand(0), Node);
5349 ModeVT, dl, Chain, StackPtr,
5359 EVT ModeVT =
Mode.getValueType();
5363 Node->getOperand(0), dl,
Mode, StackPtr,
5377 Node->getOperand(0), Node));
5384 LLVM_DEBUG(
dbgs() <<
"Successfully converted node to libcall\n");
5385 ReplaceNode(Node,
Results.data());
5393 MVT EltVT,
MVT NewEltVT) {
5395 MVT MidVT = OldEltsPerNewElt == 1
5398 assert(
TLI.isTypeLegal(MidVT) &&
"unexpected");
5402void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
5405 MVT OVT =
Node->getSimpleValueType(0);
5416 OVT =
Node->getOperand(0).getSimpleValueType();
5427 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5428 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5429 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5430 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5431 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5432 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5433 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5434 OVT =
Node->getOperand(1).getSimpleValueType();
5437 OVT =
Node->getOperand(2).getSimpleValueType();
5440 SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
5443 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5444 switch (
Node->getOpcode()) {
5456 unsigned NewOpc =
Node->getOpcode();
5469 Tmp1 = DAG.
getNode(NewOpc, dl, NVT, Tmp1);
5485 auto AnyExtendedNode =
5491 auto LeftShiftResult =
5495 auto CTLZResult = DAG.
getNode(
Node->getOpcode(), dl, NVT, LeftShiftResult);
5517 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5528 PromoteLegalFP_TO_INT(Node, dl,
Results);
5532 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));
5538 PromoteLegalINT_TO_FP(Node, dl,
Results);
5549 &&
"VAARG promotion is supported only for vectors or integer types");
5554 Tmp1 = DAG.
getVAArg(NVT, dl, Chain, Ptr,
Node->getOperand(2),
5555 Node->getConstantOperandVal(3));
5558 Tmp2 = DAG.
getNode(TruncOp, dl, OVT, Tmp1);
5565 UpdatedNodes->insert(Tmp2.
getNode());
5566 UpdatedNodes->insert(Chain.
getNode());
5583 unsigned ExtOp, TruncOp;
5590 switch (
Node->getOpcode()) {
5615 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5616 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5618 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5627 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5628 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5639 unsigned ExtOp, TruncOp;
5640 if (
Node->getValueType(0).isVector() ||
5644 }
else if (
Node->getValueType(0).isInteger()) {
5651 Tmp1 =
Node->getOperand(0);
5653 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5654 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5656 Tmp1 = DAG.
getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5658 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1);
5660 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1,
5673 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5682 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
5683 Node->getOperand(2));
5691 MVT CVT =
Node->getSimpleValueType(0);
5692 assert(CVT == OVT &&
"not handled");
5701 Tmp1 =
Node->getOperand(0);
5702 Tmp2 =
Node->getOperand(1);
5704 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5705 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5708 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5709 Tmp4 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5736 if (
Node->isStrictFPOpcode()) {
5738 std::tie(Tmp1, std::ignore) =
5740 std::tie(Tmp2, std::ignore) =
5744 SDVTList VTs = DAG.
getVTList(
Node->getValueType(0), MVT::Other);
5746 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5751 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5752 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5754 Tmp2,
Node->getOperand(2),
Node->getFlags()));
5763 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5764 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5766 Node->getOperand(0),
Node->getOperand(1),
5767 Tmp1, Tmp2,
Node->getOperand(4)));
5789 DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
Node->getFlags());
5796 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5805 SDVTList VTs = DAG.
getVTList(NVT, MVT::Other);
5807 Node->getOperand(1));
5809 Node->getOperand(2));
5830 {
Node->getOperand(0),
Node->getOperand(1)});
5832 {
Node->getOperand(0),
Node->getOperand(2)});
5835 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5836 {Tmp3, Tmp1, Tmp2});
5861 DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5866 {
Node->getOperand(0),
Node->getOperand(1)});
5868 {
Node->getOperand(0),
Node->getOperand(2)});
5870 {
Node->getOperand(0),
Node->getOperand(3)});
5873 Tmp4 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5874 {Tmp4, Tmp1, Tmp2, Tmp3});
5885 Tmp2 =
Node->getOperand(1);
5886 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5901 {
Node->getOperand(0),
Node->getOperand(1)});
5902 Tmp2 =
Node->getOperand(2);
5914 {
Node->getOperand(0),
Node->getOperand(1)});
5915 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5916 {Tmp1.getValue(1), Tmp1, Node->getOperand(2)});
5940 for (
unsigned ResNum = 0; ResNum <
Node->getNumValues(); ResNum++)
5975 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1,
Node->getFlags());
5981 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
6009 {
Node->getOperand(0),
Node->getOperand(1)});
6010 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
6011 {Tmp1.getValue(1), Tmp1});
6023 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl,
Node->getValueType(0), Tmp1);
6031 {
Node->getOperand(0),
Node->getOperand(1)});
6032 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
6033 {Tmp1.getValue(1), Tmp1});
6048 "Invalid promote type for build_vector");
6081 "Invalid promote type for extract_vector_elt");
6088 EVT IdxVT =
Idx.getValueType();
6096 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
6127 "Invalid promote type for insert_vector_elt");
6135 EVT IdxVT =
Idx.getValueType();
6145 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
6150 CastVal, IdxOffset);
6153 NewVec, Elt, InEltIdx);
6205 "unexpected promotion type");
6207 "unexpected atomic_swap with illegal type");
6247 "unexpected promotion type");
6249 "unexpected atomic_load with illegal type");
6260 MVT ScalarType =
Scalar.getSimpleValueType();
6264 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
6269 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
6281 case ISD::VP_REDUCE_FMAX:
6282 case ISD::VP_REDUCE_FMIN:
6283 case ISD::VP_REDUCE_FMAXIMUM:
6284 case ISD::VP_REDUCE_FMINIMUM:
6285 Results.push_back(PromoteReduction(Node));
6292 ReplaceNode(Node,
Results.data());
6310 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes);
6317 bool AnyLegalized =
false;
6328 if (LegalizedNodes.
insert(
N).second) {
6329 AnyLegalized =
true;
6350 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes, &UpdatedNodes);
6357 return LegalizedNodes.
count(
N);
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
bpf unwind BPF Unwind Lower
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool isSigned(unsigned Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
static MaybeAlign getAlign(Value *Ptr)
static constexpr Value * getValue(Ty &ValueOrUse)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
static bool isSinCosLibcallAvailable(SDNode *Node, const LibcallLoweringInfo &Libcalls)
Return true if sincos or __sincos_stret libcall is available.
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
static bool canUseFastMathLibcall(const SDNode *Node)
Return if we can use the FAST_* variant of a math libcall for the node.
static MachineMemOperand * getStackAlignedMMO(SDValue StackPtr, MachineFunction &MF, bool isObjectScalable)
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
Promote Memory to Register
PowerPC Reduce CR logical Operation
static constexpr MCPhysReg SPReg
SmallVector< MachineOperand, 4 > Cond
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
const SDValue & getVal() const
LLVM_ABI Type * getStructRetType() const
static LLVM_ABI bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const ConstantInt * getConstantIntValue() const
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
bool isLittleEndian() const
Layout endianness...
unsigned getAllocaAddrSpace() const
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
const BasicBlock & back() const
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Tracks which library functions to use for a particular subtarget or function.
const RTLIB::RuntimeLibcallsInfo & getRuntimeLibcallsInfo() const
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Return the lowering's selection of implementation call for Call.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
LLVM_ABI unsigned getEntrySize(const DataLayout &TD) const
getEntrySize - Return the size of each entry in the jump table.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOStore
The memory access writes data.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
bool isStrictFPOpcode()
Test if this node is a strict floating point pseudo-op.
ArrayRef< SDUse > ops() const
LLVM_ABI void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
iterator_range< user_iterator > users()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
ISD::CondCode getOperandCondCode(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op)
Return the specified value casted to the target's desired shift amount type.
LLVM_ABI SDValue emitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, const SDLoc &DL, SDValue Chain)
Emit a store/load combination to the stack.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
bool isKnownNeverSNaN(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
LLVM_ABI SDValue makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, SDValue InChain, SDNode *Node)
Helper used to make a call to a library function that has one argument of pointer type.
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI SDValue getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO)
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false, SDNodeFlags Flags={})
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
LLVM_ABI bool shouldOptForSize() const
LLVM_ABI bool hasSwiftErrorArg() const
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
LLVM_ABI SDValue expandVACopy(SDNode *Node)
Expand the specified ISD::VACOPY node as the Legalize pass would.
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
allnodes_const_iterator allnodes_end() const
LLVM_ABI void DeleteNode(SDNode *N)
Remove the specified node from the system.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue expandVAArg(SDNode *Node)
Expand the specified ISD::VAARG node as the Legalize pass would.
LLVM_ABI void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
LLVM_ABI SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
LLVM_ABI void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
LLVM_ABI std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
LLVM_ABI unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
const TargetLibraryInfo & getLibInfo() const
LLVM_ABI SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
LLVM_ABI SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool shouldExpandBuildVectorWithShuffles(EVT, unsigned DefinedValues) const
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const
Returns true if the specified FP immediate can be materialized without a load from the constant pool.
bool isOperationLegalOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal using promotion.
MVT getRegisterType(LLVMContext &Context, EVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getCondCodeAction(ISD::CondCode CC, MVT VT) const
Return how the condition code should be treated: either it is legal, needs to be expanded to some oth...
LegalizeAction getTruncStoreAction(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace) const
Return how this store with truncation should be treated: either it is legal, needs to be promoted to ...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
bool isStrictFPEnabled() const
Return true if the target support strict float operation.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
LegalizeAction getVectorInterleaveAction(unsigned Opc, unsigned Factor, EVT VT) const
Return how a VECTOR_INTERLEAVE or VECTOR_DEINTERLEAVE node with the given interleave factor and VT sh...
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal for a comparison of the specified types on this ...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual bool isJumpTableRelative() const
virtual bool ShouldShrinkFPConstant(EVT) const
If true, then instruction selection should seek to shrink the FP constant of the specified type to a ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual LegalizeAction getCustomOperationAction(SDNode &Op) const
How to legalize this custom operation?
LegalizeAction getLoadAction(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, unsigned ExtType, bool Atomic) const
Return how this load with extension should be treated: either it is legal, needs to be promoted to a ...
virtual FPImmAction getFPImmAction(const APFloat &, EVT, bool ForCodeSize=false) const
Returns how the legalizer should materialize the specified FP immediate.
LegalizeAction getStrictFPOperationAction(unsigned Op, EVT VT) const
virtual bool useSoftFloat() const
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const
Returns true if arguments should be sign-extended in lib calls.
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isTruncStoreLegalOrCustom(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace) const
Return true if the specified store with truncation has solution on this target.
bool isCondCodeLegalOrCustom(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal or custom for a comparison of the specified type...
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
bool isLoadLegal(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, unsigned ExtType, bool Atomic) const
Return true if the specified load with extension is legal on this target.
bool isLoadLegalOrCustom(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, unsigned ExtType, bool Atomic) const
Return true if the specified load with extension is legal or custom on this target.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
MVT getTypeToPromoteTo(unsigned Op, MVT VT) const
If the action for this operation is to promote, this method returns the ValueType to promote to.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
void expandVectorDeinterleave(SDNode *Node, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
Expand a VECTOR_DEINTERLEAVE without introducing wider vector types.
bool expandMultipleResultFPLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, SDNode *Node, SmallVectorImpl< SDValue > &Results, std::optional< unsigned > CallRetResNo={}) const
Expands a node with multiple results to an FP or vector libcall.
bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]MULO.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
bool LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, bool &NeedInvert, const SDLoc &dl, SDValue &Chain, bool IsSignaling=false) const
Legalize a SETCC with given LHS and RHS and condition code CC on the current target.
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, const SDValue LHS, const SDValue RHS, SDValue &Lo, SDValue &Hi) const
Calculate full product of LHS and RHS either via a libcall or through brute force expansion of the mu...
SDValue expandFCANONICALIZE(SDNode *Node, SelectionDAG &DAG) const
Expand FCANONICALIZE to FMUL with 1.
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_POISON nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_POISON nodes.
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
SDValue expandABD(SDNode *N, SelectionDAG &DAG) const
Expand ABDS/ABDU nodes.
SDValue expandCLMUL(SDNode *N, SelectionDAG &DAG) const
Expand carryless multiply.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
void expandSADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::S(ADD|SUB)O.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue expandVPCTTZElements(SDNode *N, SelectionDAG &DAG) const
Expand VP_CTTZ_ELTS/VP_CTTZ_ELTS_ZERO_POISON nodes.
bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const
Expand float to UINT conversion.
bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const
Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
SDValue expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminimumnum/fmaximumnum into multiple comparison with selects.
void expandVectorInterleave(SDNode *Node, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
Expand a VECTOR_INTERLEAVE without introducing wider vector types.
SDValue expandVectorSplice(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::VECTOR_SPLICE.
SDValue getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, EVT SubVecVT, SDValue Index, const SDNodeFlags PtrArithFlags=SDNodeFlags()) const
Get a pointer to a sub-vector of type SubVecVT at index Idx located in memory for a vector of type Ve...
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
SDValue expandFMINIMUM_FMAXIMUM(SDNode *N, SelectionDAG &DAG) const
Expand fminimum/fmaximum into multiple comparison with selects.
bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const
Expand float(f32) to SINT(i64) conversion.
virtual SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const
Returns relocation base for the given PIC jumptable.
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
bool isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, SDValue &Chain) const
Check whether a given call node is in tail position within its function.
SDValue expandCONVERT_TO_ARBITRARY_FP(SDNode *Node, SelectionDAG &DAG) const
Expand CONVERT_TO_ARBITRARY_FP using bit manipulation.
SDValue expandFunnelShift(SDNode *N, SelectionDAG &DAG) const
Expand funnel shift.
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandPEXT(SDNode *N, SelectionDAG &DAG) const
Expand parallel bit extract (compress).
bool expandMODF(SDNode *N, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results) const
Expand MODF node into fractional and integer parts.
SDValue expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const
Expand round(fp) to fp conversion.
SDValue expandCONVERT_FROM_ARBITRARY_FP(SDNode *Node, SelectionDAG &DAG) const
Expand CONVERT_FROM_ARBITRARY_FP using bit manipulation.
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index, const SDNodeFlags PtrArithFlags=SDNodeFlags()) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCMP(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]CMP.
SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[U|S]MULFIX[SAT].
void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::U(ADD|SUB)O.
SDValue expandPDEP(SDNode *N, SelectionDAG &DAG) const
Expand parallel bit deposit (expand).
bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const
Expand UINT(i64) to double(f64) conversion.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
SDValue expandAVG(SDNode *N, SelectionDAG &DAG) const
Expand vector/scalar AVGCEILS/AVGCEILU/AVGFLOORS/AVGFLOORU nodes.
SDValue expandCTLS(SDNode *N, SelectionDAG &DAG) const
Expand CTLS (count leading sign bits) nodes.
Primary interface to the complete machine description for the target machine.
const Triple & getTargetTriple() const
virtual const TargetFrameLowering * getFrameLowering() const
static constexpr TypeSize getFixed(ScalarTy ExactSize)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isVoidTy() const
Return true if this is 'void'.
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
constexpr ScalarTy getFixedValue() const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ POISON
POISON - A poison node.
@ SET_FPENV
Sets the current floating-point environment.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ STACKADDRESS
STACKADDRESS - Represents the llvm.stackaddress intrinsic.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ FRAME_TO_ARGS_OFFSET
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
@ RESET_FPENV
Set floating-point environment to default state.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ FMODF
FMODF - Decomposes the operand into integral and fractional parts, each having the same type and sign...
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ FSINCOSPI
FSINCOSPI - Compute both the sine and cosine times pi more accurately than FSINCOS(pi*x),...
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ CLMUL
Carry-less multiplication operations.
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ CONVERT_FROM_ARBITRARY_FP
CONVERT_FROM_ARBITRARY_FP - This operator converts from an arbitrary floating-point represented as an...
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ VECREDUCE_FMAXIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM nodes do not propagate NaNs and order signed zeroes using the llvm....
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ CTLS
Count leading redundant sign bits.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ GLOBAL_OFFSET_TABLE
The address of the GOT.
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ CLEAR_CACHE
llvm.clear_cache intrinsic Operands: Input Chain, Start Addres, End Address Outputs: Output Chain
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ EXPERIMENTAL_VECTOR_HISTOGRAM
Experimental vector histogram intrinsic Operands: Input Chain, Inc, Mask, Base, Index,...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ CONVERT_TO_ARBITRARY_FP
CONVERT_TO_ARBITRARY_FP - Converts a native FP value to an arbitrary floating-point format,...
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ SET_FPENV_MEM
Sets the current floating point environment.
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
LLVM_ABI NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
LLVM_ABI std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
LLVM_ABI bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
LLVM_ABI Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI AtomicLibcall selectAtomicLibcall(const LibcallLoweringInfo &Libcalls, unsigned Opc, AtomicOrdering Order, MVT VT)
Lower to outline atomic libcall if outline atomics enabled, or to sync libcall otherwise.
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
NodeAddr< NodeBase * > Node
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
@ Undef
Value of the register doesn't matter.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
@ Load
The value being inserted comes from a load (InsertElement only).
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
Returns: X * 2^Exp for integral exponents.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
AtomicOrdering
Atomic ordering for LLVM's memory model.
To bit_cast(const From &from) noexcept
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
@ Fast
Assign the register banks as fast as possible (default).
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
LLVM_ABI unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall.
LLVM_ABI std::pair< FunctionType *, AttributeList > getFunctionTy(LLVMContext &Ctx, const Triple &TT, const DataLayout &DL, RTLIB::LibcallImpl LibcallImpl) const
These are IR-level optimization flags that may be propagated to SDNodes.
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
bool IsPostTypeLegalization
MakeLibCallOptions & setIsSigned(bool Value=true)