99void SelectionDAG::DAGNodeDeletedListener::anchor() {}
100void SelectionDAG::DAGNodeInsertedListener::anchor() {}
102#define DEBUG_TYPE "selectiondag"
106 cl::desc(
"Gang up loads and stores generated by inlining of memcpy"));
109 cl::desc(
"Number limit for gluing ld/st of memcpy."),
125 return getValueAPF().bitwiseIsEqual(V);
148 N->getValueType(0).getVectorElementType().getSizeInBits();
149 if (
auto *Op0 = dyn_cast<ConstantSDNode>(
N->getOperand(0))) {
150 SplatVal = Op0->getAPIntValue().
trunc(EltSize);
153 if (
auto *Op0 = dyn_cast<ConstantFPSDNode>(
N->getOperand(0))) {
154 SplatVal = Op0->getValueAPF().bitcastToAPInt().
trunc(EltSize);
159 auto *BV = dyn_cast<BuildVectorSDNode>(
N);
164 unsigned SplatBitSize;
166 unsigned EltSize =
N->getValueType(0).getVectorElementType().getSizeInBits();
171 const bool IsBigEndian =
false;
172 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
173 EltSize, IsBigEndian) &&
174 EltSize == SplatBitSize;
183 N =
N->getOperand(0).getNode();
192 unsigned i = 0, e =
N->getNumOperands();
195 while (i != e &&
N->getOperand(i).isUndef())
199 if (i == e)
return false;
210 unsigned EltSize =
N->getValueType(0).getScalarSizeInBits();
212 if (CN->getAPIntValue().countr_one() < EltSize)
215 if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize)
223 for (++i; i != e; ++i)
224 if (
N->getOperand(i) != NotZero && !
N->getOperand(i).isUndef())
232 N =
N->getOperand(0).getNode();
241 bool IsAllUndef =
true;
254 unsigned EltSize =
N->getValueType(0).getScalarSizeInBits();
256 if (CN->getAPIntValue().countr_zero() < EltSize)
259 if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize)
286 if (!isa<ConstantSDNode>(
Op))
299 if (!isa<ConstantFPSDNode>(
Op))
307 assert(
N->getValueType(0).isVector() &&
"Expected a vector!");
309 unsigned EltSize =
N->getValueType(0).getScalarSizeInBits();
310 if (EltSize <= NewEltSize)
314 return (
N->getOperand(0).getValueType().getScalarSizeInBits() <=
319 return (
N->getOperand(0).getValueType().getScalarSizeInBits() <=
329 if (!isa<ConstantSDNode>(
Op))
332 APInt C =
Op->getAsAPIntVal().trunc(EltSize);
333 if (
Signed &&
C.trunc(NewEltSize).sext(EltSize) !=
C)
335 if (!
Signed &&
C.trunc(NewEltSize).zext(EltSize) !=
C)
346 if (
N->getNumOperands() == 0)
352 return N->getOpcode() ==
ISD::FREEZE &&
N->getOperand(0).isUndef();
355template <
typename ConstNodeType>
357 std::function<
bool(ConstNodeType *)>
Match,
360 if (
auto *
C = dyn_cast<ConstNodeType>(
Op))
368 EVT SVT =
Op.getValueType().getScalarType();
370 if (AllowUndefs &&
Op.getOperand(i).isUndef()) {
376 auto *Cst = dyn_cast<ConstNodeType>(
Op.getOperand(i));
377 if (!Cst || Cst->getValueType(0) != SVT || !
Match(Cst))
383template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
385template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
391 bool AllowUndefs,
bool AllowTypeMismatch) {
392 if (!AllowTypeMismatch &&
LHS.getValueType() !=
RHS.getValueType())
396 if (
auto *LHSCst = dyn_cast<ConstantSDNode>(
LHS))
397 if (
auto *RHSCst = dyn_cast<ConstantSDNode>(
RHS))
398 return Match(LHSCst, RHSCst);
401 if (
LHS.getOpcode() !=
RHS.getOpcode() ||
406 EVT SVT =
LHS.getValueType().getScalarType();
407 for (
unsigned i = 0, e =
LHS.getNumOperands(); i != e; ++i) {
410 bool LHSUndef = AllowUndefs && LHSOp.
isUndef();
411 bool RHSUndef = AllowUndefs && RHSOp.
isUndef();
412 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
413 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
414 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
416 if (!AllowTypeMismatch && (LHSOp.
getValueType() != SVT ||
419 if (!
Match(LHSCst, RHSCst))
426 switch (VecReduceOpcode) {
431 case ISD::VP_REDUCE_FADD:
432 case ISD::VP_REDUCE_SEQ_FADD:
436 case ISD::VP_REDUCE_FMUL:
437 case ISD::VP_REDUCE_SEQ_FMUL:
440 case ISD::VP_REDUCE_ADD:
443 case ISD::VP_REDUCE_MUL:
446 case ISD::VP_REDUCE_AND:
449 case ISD::VP_REDUCE_OR:
452 case ISD::VP_REDUCE_XOR:
455 case ISD::VP_REDUCE_SMAX:
458 case ISD::VP_REDUCE_SMIN:
461 case ISD::VP_REDUCE_UMAX:
464 case ISD::VP_REDUCE_UMIN:
467 case ISD::VP_REDUCE_FMAX:
470 case ISD::VP_REDUCE_FMIN:
473 case ISD::VP_REDUCE_FMAXIMUM:
476 case ISD::VP_REDUCE_FMINIMUM:
485#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
488#include "llvm/IR/VPIntrinsics.def"
496#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
497#define VP_PROPERTY_BINARYOP return true;
498#define END_REGISTER_VP_SDNODE(VPSD) break;
499#include "llvm/IR/VPIntrinsics.def"
508#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
509#define VP_PROPERTY_REDUCTION(STARTPOS, ...) return true;
510#define END_REGISTER_VP_SDNODE(VPSD) break;
511#include "llvm/IR/VPIntrinsics.def"
521#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
524#include "llvm/IR/VPIntrinsics.def"
533#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
536#include "llvm/IR/VPIntrinsics.def"
546#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
547#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
548#define END_REGISTER_VP_SDNODE(VPOPC) break;
549#include "llvm/IR/VPIntrinsics.def"
558#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
559#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
560#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
561#include "llvm/IR/VPIntrinsics.def"
608 bool isIntegerLike) {
633 bool IsInteger =
Type.isInteger();
638 unsigned Op = Op1 | Op2;
654 bool IsInteger =
Type.isInteger();
689 ID.AddPointer(VTList.
VTs);
695 for (
const auto &
Op : Ops) {
696 ID.AddPointer(
Op.getNode());
697 ID.AddInteger(
Op.getResNo());
704 for (
const auto &
Op : Ops) {
705 ID.AddPointer(
Op.getNode());
706 ID.AddInteger(
Op.getResNo());
719 switch (
N->getOpcode()) {
728 ID.AddPointer(
C->getConstantIntValue());
729 ID.AddBoolean(
C->isOpaque());
734 ID.AddPointer(cast<ConstantFPSDNode>(
N)->getConstantFPValue());
750 ID.AddInteger(cast<RegisterSDNode>(
N)->
getReg());
753 ID.AddPointer(cast<RegisterMaskSDNode>(
N)->getRegMask());
756 ID.AddPointer(cast<SrcValueSDNode>(
N)->getValue());
760 ID.AddInteger(cast<FrameIndexSDNode>(
N)->getIndex());
764 if (cast<LifetimeSDNode>(
N)->hasOffset()) {
765 ID.AddInteger(cast<LifetimeSDNode>(
N)->
getSize());
770 ID.AddInteger(cast<PseudoProbeSDNode>(
N)->getGuid());
771 ID.AddInteger(cast<PseudoProbeSDNode>(
N)->getIndex());
772 ID.AddInteger(cast<PseudoProbeSDNode>(
N)->getAttributes());
776 ID.AddInteger(cast<JumpTableSDNode>(
N)->getIndex());
777 ID.AddInteger(cast<JumpTableSDNode>(
N)->getTargetFlags());
782 ID.AddInteger(CP->getAlign().value());
783 ID.AddInteger(CP->getOffset());
784 if (CP->isMachineConstantPoolEntry())
785 CP->getMachineCPVal()->addSelectionDAGCSEId(
ID);
787 ID.AddPointer(CP->getConstVal());
788 ID.AddInteger(CP->getTargetFlags());
800 ID.AddInteger(LD->getMemoryVT().getRawBits());
801 ID.AddInteger(LD->getRawSubclassData());
802 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
803 ID.AddInteger(LD->getMemOperand()->getFlags());
808 ID.AddInteger(ST->getMemoryVT().getRawBits());
809 ID.AddInteger(ST->getRawSubclassData());
810 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
811 ID.AddInteger(ST->getMemOperand()->getFlags());
822 case ISD::VP_STORE: {
830 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
837 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
844 case ISD::VP_GATHER: {
852 case ISD::VP_SCATTER: {
941 if (
auto *MN = dyn_cast<MemIntrinsicSDNode>(
N)) {
942 ID.AddInteger(MN->getRawSubclassData());
943 ID.AddInteger(MN->getPointerInfo().getAddrSpace());
944 ID.AddInteger(MN->getMemOperand()->getFlags());
945 ID.AddInteger(MN->getMemoryVT().getRawBits());
968 if (
N->getValueType(0) == MVT::Glue)
971 switch (
N->getOpcode()) {
979 for (
unsigned i = 1, e =
N->getNumValues(); i != e; ++i)
980 if (
N->getValueType(i) == MVT::Glue)
997 if (Node.use_empty())
1012 while (!DeadNodes.
empty()) {
1021 DUL->NodeDeleted(
N,
nullptr);
1024 RemoveNodeFromCSEMaps(
N);
1055 RemoveNodeFromCSEMaps(
N);
1059 DeleteNodeNotInCSEMaps(
N);
1062void SelectionDAG::DeleteNodeNotInCSEMaps(
SDNode *
N) {
1063 assert(
N->getIterator() != AllNodes.begin() &&
1064 "Cannot delete the entry node!");
1065 assert(
N->use_empty() &&
"Cannot delete a node that is not dead!");
1074 assert(!(V->isVariadic() && isParameter));
1076 ByvalParmDbgValues.push_back(V);
1078 DbgValues.push_back(V);
1079 for (
const SDNode *Node : V->getSDNodes())
1081 DbgValMap[Node].push_back(V);
1086 if (
I == DbgValMap.end())
1088 for (
auto &Val:
I->second)
1089 Val->setIsInvalidated();
1093void SelectionDAG::DeallocateNode(
SDNode *
N) {
1117 switch (
N->getOpcode()) {
1123 EVT VT =
N->getValueType(0);
1124 assert(
N->getNumValues() == 1 &&
"Too many results!");
1126 "Wrong return type!");
1127 assert(
N->getNumOperands() == 2 &&
"Wrong number of operands!");
1128 assert(
N->getOperand(0).getValueType() ==
N->getOperand(1).getValueType() &&
1129 "Mismatched operand types!");
1131 "Wrong operand type!");
1133 "Wrong return type size");
1137 assert(
N->getNumValues() == 1 &&
"Too many results!");
1138 assert(
N->getValueType(0).isVector() &&
"Wrong return type!");
1139 assert(
N->getNumOperands() ==
N->getValueType(0).getVectorNumElements() &&
1140 "Wrong number of operands!");
1141 EVT EltVT =
N->getValueType(0).getVectorElementType();
1143 assert((
Op.getValueType() == EltVT ||
1144 (EltVT.
isInteger() &&
Op.getValueType().isInteger() &&
1145 EltVT.
bitsLE(
Op.getValueType()))) &&
1146 "Wrong operand type!");
1147 assert(
Op.getValueType() ==
N->getOperand(0).getValueType() &&
1148 "Operands must all have the same type");
1160void SelectionDAG::InsertNode(
SDNode *
N) {
1161 AllNodes.push_back(
N);
1163 N->PersistentId = NextPersistentId++;
1166 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1167 DUL->NodeInserted(
N);
1174bool SelectionDAG::RemoveNodeFromCSEMaps(
SDNode *
N) {
1175 bool Erased =
false;
1176 switch (
N->getOpcode()) {
1179 assert(CondCodeNodes[cast<CondCodeSDNode>(
N)->
get()] &&
1180 "Cond code doesn't exist!");
1181 Erased = CondCodeNodes[cast<CondCodeSDNode>(
N)->get()] !=
nullptr;
1182 CondCodeNodes[cast<CondCodeSDNode>(
N)->get()] =
nullptr;
1185 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(
N)->getSymbol());
1189 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
1194 auto *MCSN = cast<MCSymbolSDNode>(
N);
1195 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1199 EVT VT = cast<VTSDNode>(
N)->getVT();
1201 Erased = ExtendedValueTypeNodes.erase(VT);
1212 Erased = CSEMap.RemoveNode(
N);
1219 if (!Erased &&
N->getValueType(
N->getNumValues()-1) != MVT::Glue &&
1234SelectionDAG::AddModifiedNodeToCSEMaps(
SDNode *
N) {
1238 SDNode *Existing = CSEMap.GetOrInsertNode(
N);
1239 if (Existing !=
N) {
1247 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1248 DUL->NodeDeleted(
N, Existing);
1249 DeleteNodeNotInCSEMaps(
N);
1255 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1256 DUL->NodeUpdated(
N);
1274 Node->intersectFlagsWith(
N->getFlags());
1294 Node->intersectFlagsWith(
N->getFlags());
1312 Node->intersectFlagsWith(
N->getFlags());
1325 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0,
DebugLoc(),
1328 InsertNode(&EntryNode);
1339 SDAGISelPass = PassPtr;
1343 LibInfo = LibraryInfo;
1349 FnVarLocs = VarLocs;
1353 assert(!UpdateListeners &&
"Dangling registered DAGUpdateListeners");
1355 OperandRecycler.clear(OperandAllocator);
1364void SelectionDAG::allnodes_clear() {
1365 assert(&*AllNodes.begin() == &EntryNode);
1366 AllNodes.remove(AllNodes.begin());
1367 while (!AllNodes.empty())
1368 DeallocateNode(&AllNodes.front());
1370 NextPersistentId = 0;
1376 SDNode *
N = CSEMap.FindNodeOrInsertPos(
ID, InsertPos);
1378 switch (
N->getOpcode()) {
1383 "debug location. Use another overload.");
1390 const SDLoc &
DL,
void *&InsertPos) {
1391 SDNode *
N = CSEMap.FindNodeOrInsertPos(
ID, InsertPos);
1393 switch (
N->getOpcode()) {
1399 if (
N->getDebugLoc() !=
DL.getDebugLoc())
1406 if (
DL.getIROrder() &&
DL.getIROrder() <
N->getIROrder())
1407 N->setDebugLoc(
DL.getDebugLoc());
1416 OperandRecycler.clear(OperandAllocator);
1417 OperandAllocator.
Reset();
1420 ExtendedValueTypeNodes.clear();
1421 ExternalSymbols.clear();
1422 TargetExternalSymbols.clear();
1425 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
nullptr);
1426 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
nullptr);
1428 EntryNode.UseList =
nullptr;
1429 InsertNode(&EntryNode);
1435 return VT.
bitsGT(
Op.getValueType())
1441std::pair<SDValue, SDValue>
1445 "Strict no-op FP extend/round not allowed.");
1452 return std::pair<SDValue, SDValue>(Res,
SDValue(Res.
getNode(), 1));
1456 return VT.
bitsGT(
Op.getValueType()) ?
1462 return VT.
bitsGT(
Op.getValueType()) ?
1468 return VT.
bitsGT(
Op.getValueType()) ?
1476 auto Type =
Op.getValueType();
1480 auto Size =
Op.getValueSizeInBits();
1491 auto Type =
Op.getValueType();
1495 auto Size =
Op.getValueSizeInBits();
1506 auto Type =
Op.getValueType();
1510 auto Size =
Op.getValueSizeInBits();
1528 EVT OpVT =
Op.getValueType();
1530 "Cannot getZeroExtendInReg FP types");
1532 "getZeroExtendInReg type should be vector iff the operand "
1536 "Vector element counts must match in getZeroExtendInReg");
1548 EVT OpVT =
Op.getValueType();
1550 "Cannot getVPZeroExtendInReg FP types");
1552 "getVPZeroExtendInReg type and operand type should be vector!");
1554 "Vector element counts must match in getZeroExtendInReg");
1593 return getNode(ISD::VP_XOR,
DL, VT, Val, TrueValue, Mask, EVL);
1604 return getNode(ISD::VP_ZERO_EXTEND,
DL, VT,
Op, Mask, EVL);
1606 return getNode(ISD::VP_TRUNCATE,
DL, VT,
Op, Mask, EVL);
1626 bool isT,
bool isO) {
1630 "getConstant with a uint64_t value that doesn't fit in the type!");
1635 bool isT,
bool isO) {
1636 return getConstant(*ConstantInt::get(*Context, Val),
DL, VT, isT, isO);
1640 EVT VT,
bool isT,
bool isO) {
1658 Elt = ConstantInt::get(*
getContext(), NewVal);
1677 "Can only handle an even split!");
1681 for (
unsigned i = 0; i != Parts; ++i)
1683 NewVal.
extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits),
DL,
1684 ViaEltVT, isT, isO));
1689 unsigned ViaVecNumElts = VT.
getSizeInBits() / ViaEltSizeInBits;
1700 NewVal.
extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits),
DL,
1701 ViaEltVT, isT, isO));
1706 std::reverse(EltParts.
begin(), EltParts.
end());
1725 "APInt size does not match type size!");
1734 if ((
N = FindNodeOrInsertPos(
ID,
DL, IP)))
1739 N = newSDNode<ConstantSDNode>(isT, isO, Elt, VTs);
1740 CSEMap.InsertNode(
N, IP);
1780 EVT VT,
bool isTarget) {
1795 if ((
N = FindNodeOrInsertPos(
ID,
DL, IP)))
1800 N = newSDNode<ConstantFPSDNode>(isTarget, &V, VTs);
1801 CSEMap.InsertNode(
N, IP);
1815 if (EltVT == MVT::f32)
1817 if (EltVT == MVT::f64)
1819 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1820 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1831 EVT VT, int64_t
Offset,
bool isTargetGA,
1832 unsigned TargetFlags) {
1833 assert((TargetFlags == 0 || isTargetGA) &&
1834 "Cannot set target flags on target-independent globals");
1852 ID.AddInteger(TargetFlags);
1854 if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP))
1857 auto *
N = newSDNode<GlobalAddressSDNode>(
1858 Opc,
DL.getIROrder(),
DL.getDebugLoc(), GV, VTs,
Offset, TargetFlags);
1859 CSEMap.InsertNode(
N, IP);
1871 if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
1874 auto *
N = newSDNode<FrameIndexSDNode>(FI, VTs, isTarget);
1875 CSEMap.InsertNode(
N, IP);
1881 unsigned TargetFlags) {
1882 assert((TargetFlags == 0 || isTarget) &&
1883 "Cannot set target flags on target-independent jump tables");
1889 ID.AddInteger(TargetFlags);
1891 if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
1894 auto *
N = newSDNode<JumpTableSDNode>(JTI, VTs, isTarget, TargetFlags);
1895 CSEMap.InsertNode(
N, IP);
1909 bool isTarget,
unsigned TargetFlags) {
1910 assert((TargetFlags == 0 || isTarget) &&
1911 "Cannot set target flags on target-independent globals");
1920 ID.AddInteger(Alignment->value());
1923 ID.AddInteger(TargetFlags);
1925 if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
1928 auto *
N = newSDNode<ConstantPoolSDNode>(isTarget,
C, VTs,
Offset, *Alignment,
1930 CSEMap.InsertNode(
N, IP);
1939 bool isTarget,
unsigned TargetFlags) {
1940 assert((TargetFlags == 0 || isTarget) &&
1941 "Cannot set target flags on target-independent globals");
1948 ID.AddInteger(Alignment->value());
1950 C->addSelectionDAGCSEId(
ID);
1951 ID.AddInteger(TargetFlags);
1953 if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
1956 auto *
N = newSDNode<ConstantPoolSDNode>(isTarget,
C, VTs,
Offset, *Alignment,
1958 CSEMap.InsertNode(
N, IP);
1968 if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
1971 auto *
N = newSDNode<BasicBlockSDNode>(
MBB);
1972 CSEMap.InsertNode(
N, IP);
1979 ValueTypeNodes.size())
1986 N = newSDNode<VTSDNode>(VT);
1994 N = newSDNode<ExternalSymbolSDNode>(
false,
Sym, 0,
getVTList(VT));
2009 unsigned TargetFlags) {
2011 TargetExternalSymbols[std::pair<std::string, unsigned>(
Sym, TargetFlags)];
2013 N = newSDNode<ExternalSymbolSDNode>(
true,
Sym, TargetFlags,
getVTList(VT));
2019 if ((
unsigned)
Cond >= CondCodeNodes.size())
2020 CondCodeNodes.resize(
Cond+1);
2022 if (!CondCodeNodes[
Cond]) {
2023 auto *
N = newSDNode<CondCodeSDNode>(
Cond);
2024 CondCodeNodes[
Cond] =
N;
2032 bool ConstantFold) {
2034 "APInt size does not match type size!");
2051 bool ConstantFold) {
2052 if (EC.isScalable())
2065 const APInt &StepVal) {
2089 "Must have the same number of vector elements as mask elements!");
2091 "Invalid VECTOR_SHUFFLE");
2099 int NElts = Mask.size();
2101 [&](
int M) {
return M < (NElts * 2) && M >= -1; }) &&
2102 "Index out of range");
2110 for (
int i = 0; i != NElts; ++i)
2111 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2127 for (
int i = 0; i < NElts; ++i) {
2128 if (MaskVec[i] <
Offset || MaskVec[i] >= (
Offset + NElts))
2132 if (UndefElements[MaskVec[i] -
Offset]) {
2138 if (!UndefElements[i])
2142 if (
auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
2143 BlendSplat(N1BV, 0);
2144 if (
auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
2145 BlendSplat(N2BV, NElts);
2150 bool AllLHS =
true, AllRHS =
true;
2152 for (
int i = 0; i != NElts; ++i) {
2153 if (MaskVec[i] >= NElts) {
2158 }
else if (MaskVec[i] >= 0) {
2162 if (AllLHS && AllRHS)
2164 if (AllLHS && !N2Undef)
2177 bool Identity =
true, AllSame =
true;
2178 for (
int i = 0; i != NElts; ++i) {
2179 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity =
false;
2180 if (MaskVec[i] != MaskVec[0]) AllSame =
false;
2182 if (Identity && NElts)
2192 V = V->getOperand(0);
2195 if (
auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2215 if (AllSame && SameNumElts) {
2216 EVT BuildVT = BV->getValueType(0);
2233 for (
int i = 0; i != NElts; ++i)
2234 ID.AddInteger(MaskVec[i]);
2237 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
2243 int *MaskAlloc = OperandAllocator.
Allocate<
int>(NElts);
2246 auto *
N = newSDNode<ShuffleVectorSDNode>(VTs, dl.
getIROrder(),
2248 createOperands(
N, Ops);
2250 CSEMap.InsertNode(
N, IP);
2271 ID.AddInteger(RegNo);
2273 if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2276 auto *
N = newSDNode<RegisterSDNode>(RegNo, VTs);
2278 CSEMap.InsertNode(
N, IP);
2286 ID.AddPointer(RegMask);
2288 if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2291 auto *
N = newSDNode<RegisterMaskSDNode>(RegMask);
2292 CSEMap.InsertNode(
N, IP);
2307 ID.AddPointer(Label);
2309 if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2314 createOperands(
N, Ops);
2316 CSEMap.InsertNode(
N, IP);
2322 int64_t
Offset,
bool isTarget,
2323 unsigned TargetFlags) {
2331 ID.AddInteger(TargetFlags);
2333 if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2336 auto *
N = newSDNode<BlockAddressSDNode>(Opc, VTs, BA,
Offset, TargetFlags);
2337 CSEMap.InsertNode(
N, IP);
2348 if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2351 auto *
N = newSDNode<SrcValueSDNode>(V);
2352 CSEMap.InsertNode(
N, IP);
2363 if (
SDNode *E = FindNodeOrInsertPos(
ID, IP))
2366 auto *
N = newSDNode<MDNodeSDNode>(MD);
2367 CSEMap.InsertNode(
N, IP);
2373 if (VT == V.getValueType())
2380 unsigned SrcAS,
unsigned DestAS) {
2385 ID.AddInteger(SrcAS);
2386 ID.AddInteger(DestAS);
2389 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
2393 VTs, SrcAS, DestAS);
2394 createOperands(
N, Ops);
2396 CSEMap.InsertNode(
N, IP);
2408 EVT OpTy =
Op.getValueType();
2410 if (OpTy == ShTy || OpTy.
isVector())
return Op;
2418 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2419 EVT VT = Node->getValueType(0);
2420 SDValue Tmp1 = Node->getOperand(0);
2421 SDValue Tmp2 = Node->getOperand(1);
2422 const MaybeAlign MA(Node->getConstantOperandVal(3));
2454 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2455 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2466 Align RedAlign = UseABI ?
DL.getABITypeAlign(Ty) :
DL.getPrefTypeAlign(Ty);
2476 if (RedAlign > StackAlign) {
2479 unsigned NumIntermediates;
2481 NumIntermediates, RegisterVT);
2483 Align RedAlign2 = UseABI ?
DL.getABITypeAlign(Ty) :
DL.getPrefTypeAlign(Ty);
2484 if (RedAlign2 < RedAlign)
2485 RedAlign = RedAlign2;
2490 RedAlign = std::min(RedAlign, StackAlign);
2505 false,
nullptr, StackID);
2520 "Don't know how to choose the maximum size when creating a stack "
2529 Align Align = std::max(
DL.getPrefTypeAlign(Ty1),
DL.getPrefTypeAlign(Ty2));
2537 auto GetUndefBooleanConstant = [&]() {
2576 return GetUndefBooleanConstant();
2581 return GetUndefBooleanConstant();
2590 const APInt &C2 = N2C->getAPIntValue();
2592 const APInt &C1 = N1C->getAPIntValue();
2599 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2600 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2602 if (N1CFP && N2CFP) {
2607 return GetUndefBooleanConstant();
2612 return GetUndefBooleanConstant();
2618 return GetUndefBooleanConstant();
2623 return GetUndefBooleanConstant();
2628 return GetUndefBooleanConstant();
2634 return GetUndefBooleanConstant();
2663 return getSetCC(dl, VT, N2, N1, SwappedCond);
2664 }
else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2679 return GetUndefBooleanConstant();
2690 unsigned BitWidth =
Op.getScalarValueSizeInBits();
2698 unsigned Depth)
const {
2706 const APInt &DemandedElts,
2707 unsigned Depth)
const {
2714 unsigned Depth )
const {
2720 unsigned Depth)
const {
2725 const APInt &DemandedElts,
2726 unsigned Depth)
const {
2727 EVT VT =
Op.getValueType();
2734 for (
unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2735 if (!DemandedElts[EltIdx])
2739 KnownZeroElements.
setBit(EltIdx);
2741 return KnownZeroElements;
2751 unsigned Opcode = V.getOpcode();
2752 EVT VT = V.getValueType();
2755 "scalable demanded bits are ignored");
2767 UndefElts = V.getOperand(0).isUndef()
2776 APInt UndefLHS, UndefRHS;
2781 UndefElts = UndefLHS | UndefRHS;
2811 for (
unsigned i = 0; i != NumElts; ++i) {
2817 if (!DemandedElts[i])
2819 if (Scl && Scl !=
Op)
2829 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2830 for (
int i = 0; i != (int)NumElts; ++i) {
2836 if (!DemandedElts[i])
2838 if (M < (
int)NumElts)
2841 DemandedRHS.
setBit(M - NumElts);
2853 auto CheckSplatSrc = [&](
SDValue Src,
const APInt &SrcElts) {
2855 return (SrcElts.popcount() == 1) ||
2857 (SrcElts & SrcUndefs).
isZero());
2859 if (!DemandedLHS.
isZero())
2860 return CheckSplatSrc(V.getOperand(0), DemandedLHS);
2861 return CheckSplatSrc(V.getOperand(1), DemandedRHS);
2865 SDValue Src = V.getOperand(0);
2867 if (Src.getValueType().isScalableVector())
2870 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2883 SDValue Src = V.getOperand(0);
2885 if (Src.getValueType().isScalableVector())
2887 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2889 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts);
2891 UndefElts = UndefSrcElts.
trunc(NumElts);
2897 SDValue Src = V.getOperand(0);
2898 EVT SrcVT = Src.getValueType();
2908 if ((
BitWidth % SrcBitWidth) == 0) {
2910 unsigned Scale =
BitWidth / SrcBitWidth;
2912 APInt ScaledDemandedElts =
2914 for (
unsigned I = 0;
I != Scale; ++
I) {
2918 SubDemandedElts &= ScaledDemandedElts;
2922 if (!SubUndefElts.
isZero())
2936 EVT VT = V.getValueType();
2946 (AllowUndefs || !UndefElts);
2952 EVT VT = V.getValueType();
2953 unsigned Opcode = V.getOpcode();
2974 SplatIdx = (UndefElts & DemandedElts).
countr_one();
2988 auto *SVN = cast<ShuffleVectorSDNode>(V);
2989 if (!SVN->isSplat())
2991 int Idx = SVN->getSplatIndex();
2992 int NumElts = V.getValueType().getVectorNumElements();
2993 SplatIdx =
Idx % NumElts;
2994 return V.getOperand(
Idx / NumElts);
3010 if (LegalSVT.
bitsLT(SVT))
3019std::optional<ConstantRange>
3021 unsigned Depth)
const {
3024 "Unknown shift node");
3026 unsigned BitWidth = V.getScalarValueSizeInBits();
3028 if (
auto *Cst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
3029 const APInt &ShAmt = Cst->getAPIntValue();
3031 return std::nullopt;
3035 if (
auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1))) {
3036 const APInt *MinAmt =
nullptr, *MaxAmt =
nullptr;
3037 for (
unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3038 if (!DemandedElts[i])
3040 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3042 MinAmt = MaxAmt =
nullptr;
3045 const APInt &ShAmt = SA->getAPIntValue();
3047 return std::nullopt;
3048 if (!MinAmt || MinAmt->
ugt(ShAmt))
3050 if (!MaxAmt || MaxAmt->ult(ShAmt))
3053 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3054 "Failed to find matching min/max shift amounts");
3055 if (MinAmt && MaxAmt)
3065 return std::nullopt;
3068std::optional<uint64_t>
3070 unsigned Depth)
const {
3073 "Unknown shift node");
3074 if (std::optional<ConstantRange> AmtRange =
3076 if (
const APInt *ShAmt = AmtRange->getSingleElement())
3077 return ShAmt->getZExtValue();
3078 return std::nullopt;
3081std::optional<uint64_t>
3083 EVT VT = V.getValueType();
3090std::optional<uint64_t>
3092 unsigned Depth)
const {
3095 "Unknown shift node");
3096 if (std::optional<ConstantRange> AmtRange =
3098 return AmtRange->getUnsignedMin().getZExtValue();
3099 return std::nullopt;
3102std::optional<uint64_t>
3104 EVT VT = V.getValueType();
3111std::optional<uint64_t>
3113 unsigned Depth)
const {
3116 "Unknown shift node");
3117 if (std::optional<ConstantRange> AmtRange =
3119 return AmtRange->getUnsignedMax().getZExtValue();
3120 return std::nullopt;
3123std::optional<uint64_t>
3125 EVT VT = V.getValueType();
3136 EVT VT =
Op.getValueType();
3151 unsigned Depth)
const {
3152 unsigned BitWidth =
Op.getScalarValueSizeInBits();
3156 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
3160 if (
auto *
C = dyn_cast<ConstantFPSDNode>(
Op)) {
3170 assert((!
Op.getValueType().isFixedLengthVector() ||
3171 NumElts ==
Op.getValueType().getVectorNumElements()) &&
3172 "Unexpected vector size");
3177 unsigned Opcode =
Op.getOpcode();
3185 "Expected SPLAT_VECTOR implicit truncation");
3192 unsigned ScalarSize =
Op.getOperand(0).getScalarValueSizeInBits();
3194 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3201 const APInt &Step =
Op.getConstantOperandAPInt(0);
3210 const APInt MinNumElts =
3216 .
umul_ov(MinNumElts, Overflow);
3220 const APInt MaxValue = (MaxNumElts - 1).
umul_ov(Step, Overflow);
3228 assert(!
Op.getValueType().isScalableVector());
3232 if (!DemandedElts[i])
3241 "Expected BUILD_VECTOR implicit truncation");
3254 assert(!
Op.getValueType().isScalableVector());
3257 APInt DemandedLHS, DemandedRHS;
3261 DemandedLHS, DemandedRHS))
3266 if (!!DemandedLHS) {
3274 if (!!DemandedRHS) {
3283 const APInt &Multiplier =
Op.getConstantOperandAPInt(0);
3288 if (
Op.getValueType().isScalableVector())
3292 EVT SubVectorVT =
Op.getOperand(0).getValueType();
3295 for (
unsigned i = 0; i != NumSubVectors; ++i) {
3297 DemandedElts.
extractBits(NumSubVectorElts, i * NumSubVectorElts);
3298 if (!!DemandedSub) {
3310 if (
Op.getValueType().isScalableVector())
3319 APInt DemandedSrcElts = DemandedElts;
3324 if (!!DemandedSubElts) {
3329 if (!!DemandedSrcElts) {
3339 if (
Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())
3342 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3348 if (
Op.getValueType().isScalableVector())
3352 if (DemandedElts != 1)
3363 if (
Op.getValueType().isScalableVector())
3383 if ((
BitWidth % SubBitWidth) == 0) {
3390 unsigned SubScale =
BitWidth / SubBitWidth;
3391 APInt SubDemandedElts(NumElts * SubScale, 0);
3392 for (
unsigned i = 0; i != NumElts; ++i)
3393 if (DemandedElts[i])
3394 SubDemandedElts.
setBit(i * SubScale);
3396 for (
unsigned i = 0; i != SubScale; ++i) {
3399 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3400 Known.
insertBits(Known2, SubBitWidth * Shifts);
3405 if ((SubBitWidth %
BitWidth) == 0) {
3406 assert(
Op.getValueType().isVector() &&
"Expected bitcast to vector");
3411 unsigned SubScale = SubBitWidth /
BitWidth;
3412 APInt SubDemandedElts =
3417 for (
unsigned i = 0; i != NumElts; ++i)
3418 if (DemandedElts[i]) {
3419 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3450 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3454 Op.getOperand(0), DemandedElts,
false,
Depth + 1);
3460 if (
Op->getFlags().hasNoSignedWrap() &&
3461 Op.getOperand(0) ==
Op.getOperand(1) &&
3488 unsigned SignBits1 =
3492 unsigned SignBits0 =
3498 assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3501 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3502 if (
Op.getResNo() == 0)
3509 assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3512 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3513 if (
Op.getResNo() == 0)
3566 if (
Op.getResNo() != 1)
3581 unsigned OpNo =
Op->isStrictFPOpcode() ? 1 : 0;
3593 bool NUW =
Op->getFlags().hasNoUnsignedWrap();
3594 bool NSW =
Op->getFlags().hasNoSignedWrap();
3601 if (std::optional<uint64_t> ShMinAmt =
3610 Op->getFlags().hasExact());
3613 if (std::optional<uint64_t> ShMinAmt =
3621 Op->getFlags().hasExact());
3626 unsigned Amt =
C->getAPIntValue().urem(
BitWidth);
3632 DemandedElts,
Depth + 1);
3657 assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3660 unsigned LoBits =
Op.getOperand(0).getScalarValueSizeInBits();
3661 unsigned HiBits =
Op.getOperand(1).getScalarValueSizeInBits();
3664 Known = Known2.
concat(Known);
3678 if (
Op.getResNo() == 0)
3686 EVT EVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
3727 !
Op.getValueType().isScalableVector()) {
3741 for (
unsigned i = 0; i != NumElts; ++i) {
3742 if (!DemandedElts[i])
3745 if (
auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3751 if (
auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3752 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3763 if (
auto *CInt = dyn_cast<ConstantInt>(Cst)) {
3765 }
else if (
auto *CFP = dyn_cast<ConstantFP>(Cst)) {
3771 }
else if (
Op.getResNo() == 0) {
3772 KnownBits Known0(!LD->getMemoryVT().isScalableVT()
3773 ? LD->getMemoryVT().getFixedSizeInBits()
3775 EVT VT =
Op.getValueType();
3782 if (
const MDNode *MD = LD->getRanges()) {
3793 if (LD->getMemoryVT().isVector())
3794 Known0 = Known0.
trunc(LD->getMemoryVT().getScalarSizeInBits());
3811 if (
Op.getValueType().isScalableVector())
3813 EVT InVT =
Op.getOperand(0).getValueType();
3825 if (
Op.getValueType().isScalableVector())
3827 EVT InVT =
Op.getOperand(0).getValueType();
3843 if (
Op.getValueType().isScalableVector())
3845 EVT InVT =
Op.getOperand(0).getValueType();
3862 EVT VT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
3865 Known.
Zero |= (~InMask);
3866 Known.
One &= (~Known.Zero);
3870 unsigned LogOfAlign =
Log2(cast<AssertAlignSDNode>(
Op)->
getAlign());
3890 Op.getOpcode() ==
ISD::ADD, Flags.hasNoSignedWrap(),
3891 Flags.hasNoUnsignedWrap(), Known, Known2);
3898 if (
Op.getResNo() == 1) {
3909 "We only compute knownbits for the difference here.");
3916 Borrow = Borrow.
trunc(1);
3930 if (
Op.getResNo() == 1) {
3941 assert(
Op.getResNo() == 0 &&
"We only compute knownbits for the sum here.");
3951 Carry = Carry.
trunc(1);
3987 const unsigned Index =
Op.getConstantOperandVal(1);
3988 const unsigned EltBitWidth =
Op.getValueSizeInBits();
3995 Known = Known.
trunc(EltBitWidth);
4011 Known = Known.
trunc(EltBitWidth);
4016 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4017 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4027 if (
Op.getValueType().isScalableVector())
4036 bool DemandedVal =
true;
4037 APInt DemandedVecElts = DemandedElts;
4038 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4039 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4040 unsigned EltIdx = CEltNo->getZExtValue();
4041 DemandedVal = !!DemandedElts[EltIdx];
4050 if (!!DemandedVecElts) {
4068 Known = Known2.
abs();
4101 if (CstLow && CstHigh) {
4106 const APInt &ValueHigh = CstHigh->getAPIntValue();
4107 if (ValueLow.
sle(ValueHigh)) {
4110 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
4133 if (IsMax && CstLow) {
4157 EVT VT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
4162 if (
Op.getResNo() == 1) {
4189 cast<AtomicSDNode>(
Op)->getMemoryVT().getScalarSizeInBits();
4191 if (
Op.getResNo() == 0) {
4215 if (
Op.getValueType().isScalableVector())
4361 return C->getAPIntValue().zextOrTrunc(
BitWidth).isPowerOf2();
4369 if (
C &&
C->getAPIntValue() == 1)
4379 if (
C &&
C->getAPIntValue().isSignMask())
4391 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
4392 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4400 if (
C->getAPIntValue().zextOrTrunc(
BitWidth).isPowerOf2())
4438 return C1->getValueAPF().getExactLog2Abs() >= 0;
4447 EVT VT =
Op.getValueType();
4459 unsigned Depth)
const {
4460 EVT VT =
Op.getValueType();
4465 unsigned FirstAnswer = 1;
4467 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
4468 const APInt &Val =
C->getAPIntValue();
4478 unsigned Opcode =
Op.getOpcode();
4482 Tmp = cast<VTSDNode>(
Op.getOperand(1))->getVT().getSizeInBits();
4483 return VTBits-Tmp+1;
4485 Tmp = cast<VTSDNode>(
Op.getOperand(1))->getVT().getSizeInBits();
4492 unsigned NumSrcBits =
Op.getOperand(0).getValueSizeInBits();
4494 if (NumSrcSignBits > (NumSrcBits - VTBits))
4495 return NumSrcSignBits - (NumSrcBits - VTBits);
4502 if (!DemandedElts[i])
4509 APInt T =
C->getAPIntValue().trunc(VTBits);
4510 Tmp2 =
T.getNumSignBits();
4514 if (
SrcOp.getValueSizeInBits() != VTBits) {
4516 "Expected BUILD_VECTOR implicit truncation");
4517 unsigned ExtraBits =
SrcOp.getValueSizeInBits() - VTBits;
4518 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4521 Tmp = std::min(Tmp, Tmp2);
4528 APInt DemandedLHS, DemandedRHS;
4532 DemandedLHS, DemandedRHS))
4535 Tmp = std::numeric_limits<unsigned>::max();
4538 if (!!DemandedRHS) {
4540 Tmp = std::min(Tmp, Tmp2);
4545 assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
4561 if (VTBits == SrcBits)
4567 if ((SrcBits % VTBits) == 0) {
4570 unsigned Scale = SrcBits / VTBits;
4571 APInt SrcDemandedElts =
4581 for (
unsigned i = 0; i != NumElts; ++i)
4582 if (DemandedElts[i]) {
4583 unsigned SubOffset = i % Scale;
4584 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4585 SubOffset = SubOffset * VTBits;
4586 if (Tmp <= SubOffset)
4588 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
4597 Tmp = cast<VTSDNode>(
Op.getOperand(1))->getVT().getScalarSizeInBits();
4598 return VTBits - Tmp + 1;
4600 Tmp = VTBits -
Op.getOperand(0).getScalarValueSizeInBits();
4604 Tmp = cast<VTSDNode>(
Op.getOperand(1))->getVT().getScalarSizeInBits();
4607 return std::max(Tmp, Tmp2);
4612 EVT SrcVT = Src.getValueType();
4620 if (std::optional<uint64_t> ShAmt =
4622 Tmp = std::min<uint64_t>(Tmp + *ShAmt, VTBits);
4625 if (std::optional<ConstantRange> ShAmtRange =
4627 uint64_t MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
4628 uint64_t MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
4636 EVT ExtVT = Ext.getValueType();
4637 SDValue Extendee = Ext.getOperand(0);
4641 if (SizeDifference <= MinShAmt) {
4642 Tmp = SizeDifference +
4645 return Tmp - MaxShAmt;
4651 return Tmp - MaxShAmt;
4661 FirstAnswer = std::min(Tmp, Tmp2);
4671 if (Tmp == 1)
return 1;
4673 return std::min(Tmp, Tmp2);
4676 if (Tmp == 1)
return 1;
4678 return std::min(Tmp, Tmp2);
4690 if (CstLow && CstHigh) {
4695 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
4696 return std::min(Tmp, Tmp2);
4705 return std::min(Tmp, Tmp2);
4713 return std::min(Tmp, Tmp2);
4717 if (
Op.getResNo() == 0 &&
Op.getOperand(0) ==
Op.getOperand(1))
4728 if (
Op.getResNo() != 1)
4742 unsigned OpNo =
Op->isStrictFPOpcode() ? 1 : 0;
4759 unsigned RotAmt =
C->getAPIntValue().urem(VTBits);
4763 RotAmt = (VTBits - RotAmt) % VTBits;
4767 if (Tmp > (RotAmt + 1))
return (Tmp - RotAmt);
4775 if (Tmp == 1)
return 1;
4780 if (CRHS->isAllOnes()) {
4786 if ((Known.
Zero | 1).isAllOnes())
4796 if (Tmp2 == 1)
return 1;
4797 return std::min(Tmp, Tmp2) - 1;
4800 if (Tmp2 == 1)
return 1;
4805 if (CLHS->isZero()) {
4810 if ((Known.
Zero | 1).isAllOnes())
4824 if (Tmp == 1)
return 1;
4825 return std::min(Tmp, Tmp2) - 1;
4829 if (SignBitsOp0 == 1)
4832 if (SignBitsOp1 == 1)
4834 unsigned OutValidBits =
4835 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
4836 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
4844 return std::min(Tmp, Tmp2);
4853 unsigned NumSrcBits =
Op.getOperand(0).getScalarValueSizeInBits();
4855 if (NumSrcSignBits > (NumSrcBits - VTBits))
4856 return NumSrcSignBits - (NumSrcBits - VTBits);
4863 const int BitWidth =
Op.getValueSizeInBits();
4864 const int Items =
Op.getOperand(0).getValueSizeInBits() /
BitWidth;
4868 const int rIndex = Items - 1 -
Op.getConstantOperandVal(1);
4883 bool DemandedVal =
true;
4884 APInt DemandedVecElts = DemandedElts;
4885 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4886 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4887 unsigned EltIdx = CEltNo->getZExtValue();
4888 DemandedVal = !!DemandedElts[EltIdx];
4891 Tmp = std::numeric_limits<unsigned>::max();
4897 Tmp = std::min(Tmp, Tmp2);
4899 if (!!DemandedVecElts) {
4901 Tmp = std::min(Tmp, Tmp2);
4903 assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
4914 const unsigned BitWidth =
Op.getValueSizeInBits();
4915 const unsigned EltBitWidth =
Op.getOperand(0).getScalarValueSizeInBits();
4927 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4928 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4938 if (Src.getValueType().isScalableVector())
4941 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4950 Tmp = std::numeric_limits<unsigned>::max();
4951 EVT SubVectorVT =
Op.getOperand(0).getValueType();
4954 for (
unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
4956 DemandedElts.
extractBits(NumSubVectorElts, i * NumSubVectorElts);
4960 Tmp = std::min(Tmp, Tmp2);
4962 assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
4975 APInt DemandedSrcElts = DemandedElts;
4978 Tmp = std::numeric_limits<unsigned>::max();
4979 if (!!DemandedSubElts) {
4984 if (!!DemandedSrcElts) {
4986 Tmp = std::min(Tmp, Tmp2);
4988 assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
4993 if (
const MDNode *Ranges = LD->getRanges()) {
4994 if (DemandedElts != 1)
4999 switch (LD->getExtensionType()) {
5034 Tmp = cast<AtomicSDNode>(
Op)->getMemoryVT().getScalarSizeInBits();
5036 if (
Op.getResNo() == 0) {
5040 return VTBits - Tmp + 1;
5042 return VTBits - Tmp;
5046 return VTBits - Tmp + 1;
5048 return VTBits - Tmp;
5056 if (
Op.getResNo() == 0) {
5059 unsigned ExtType = LD->getExtensionType();
5063 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5064 return VTBits - Tmp + 1;
5066 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5067 return VTBits - Tmp;
5072 Type *CstTy = Cst->getType();
5077 for (
unsigned i = 0; i != NumElts; ++i) {
5078 if (!DemandedElts[i])
5081 if (
auto *CInt = dyn_cast<ConstantInt>(Elt)) {
5083 Tmp = std::min(Tmp,
Value.getNumSignBits());
5086 if (
auto *CFP = dyn_cast<ConstantFP>(Elt)) {
5087 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5088 Tmp = std::min(Tmp,
Value.getNumSignBits());
5114 FirstAnswer = std::max(FirstAnswer, NumBits);
5125 unsigned Depth)
const {
5127 return Op.getScalarValueSizeInBits() - SignBits + 1;
5131 const APInt &DemandedElts,
5132 unsigned Depth)
const {
5134 return Op.getScalarValueSizeInBits() - SignBits + 1;
5138 unsigned Depth)
const {
5144 EVT VT =
Op.getValueType();
5155 const APInt &DemandedElts,
5157 unsigned Depth)
const {
5158 unsigned Opcode =
Op.getOpcode();
5185 if (!DemandedElts[i])
5194 APInt DemandedLHS, DemandedRHS;
5195 auto *SVN = cast<ShuffleVectorSDNode>(
Op);
5197 DemandedElts, DemandedLHS, DemandedRHS,
5200 if (!DemandedLHS.
isZero() &&
5204 if (!DemandedRHS.
isZero() &&
5232 return isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly, Depth + 1);
5238 unsigned Depth)
const {
5240 EVT VT =
Op.getValueType();
5253 unsigned Depth)
const {
5255 EVT VT =
Op.getValueType();
5259 if (ConsiderFlags &&
Op->hasPoisonGeneratingFlags())
5262 unsigned Opcode =
Op.getOpcode();
5300 if (
Op.getOperand(0).getValueType().isInteger())
5307 unsigned CCOp = Opcode ==
ISD::SETCC ? 2 : 4;
5308 ISD::CondCode CCCode = cast<CondCodeSDNode>(
Op.getOperand(CCOp))->get();
5309 if (((
unsigned)CCCode & 0x10U))
5340 EVT VecVT =
Op.getOperand(0).getValueType();
5352 auto *SVN = cast<ShuffleVectorSDNode>(
Op);
5354 if (Elt < 0 && DemandedElts[
Idx])
5373 unsigned Opcode =
Op.getOpcode();
5375 return Op->getFlags().hasDisjoint() ||
5397 return !
C->getValueAPF().isNaN() ||
5398 (SNaN && !
C->getValueAPF().isSignaling());
5401 unsigned Opcode =
Op.getOpcode();
5516 assert(
Op.getValueType().isFloatingPoint() &&
5517 "Floating point type expected");
5528 assert(!
Op.getValueType().isFloatingPoint() &&
5529 "Floating point types unsupported - use isKnownNeverZeroFloat");
5538 switch (
Op.getOpcode()) {
5552 if (
Op->getFlags().hasNoSignedWrap() ||
Op->getFlags().hasNoUnsignedWrap())
5556 if (ValKnown.
One[0])
5616 if (
Op->getFlags().hasExact())
5632 if (
Op->getFlags().hasExact())
5637 if (
Op->getFlags().hasNoUnsignedWrap())
5648 std::optional<bool> ne =
5655 if (
Op->getFlags().hasNoSignedWrap() ||
Op->getFlags().hasNoUnsignedWrap())
5666 const APInt &Multiplier =
Op.getConstantOperandAPInt(0);
5680 return !C1->isNegative();
5687 if (
A ==
B)
return true;
5692 if (CA->isZero() && CB->isZero())
return true;
5701 return V.getOperand(0);
5708 SDValue ExtArg = V.getOperand(0);
5727 NotOperand = NotOperand->getOperand(0);
5729 if (
Other == NotOperand)
5732 return NotOperand ==
Other->getOperand(0) ||
5733 NotOperand ==
Other->getOperand(1);
5739 A =
A->getOperand(0);
5742 B =
B->getOperand(0);
5745 return MatchNoCommonBitsPattern(
A->getOperand(0),
A->getOperand(1),
B) ||
5746 MatchNoCommonBitsPattern(
A->getOperand(1),
A->getOperand(0),
B);
5752 assert(
A.getValueType() ==
B.getValueType() &&
5753 "Values must have the same type");
5763 if (cast<ConstantSDNode>(Step)->
isZero())
5772 int NumOps = Ops.
size();
5773 assert(NumOps != 0 &&
"Can't build an empty vector!");
5775 "BUILD_VECTOR cannot be used with scalable types");
5777 "Incorrect element count in BUILD_VECTOR!");
5785 bool IsIdentity =
true;
5786 for (
int i = 0; i != NumOps; ++i) {
5788 Ops[i].getOperand(0).getValueType() != VT ||
5789 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
5790 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
5791 Ops[i].getConstantOperandAPInt(1) != i) {
5795 IdentitySrc = Ops[i].getOperand(0);
5808 assert(!Ops.
empty() &&
"Can't concatenate an empty list of vectors!");
5811 return Ops[0].getValueType() ==
Op.getValueType();
5813 "Concatenation of vectors with inconsistent value types!");
5814 assert((Ops[0].getValueType().getVectorElementCount() * Ops.
size()) ==
5816 "Incorrect element count in vector concatenation!");
5818 if (Ops.
size() == 1)
5829 bool IsIdentity =
true;
5830 for (
unsigned i = 0, e = Ops.
size(); i != e; ++i) {
5832 unsigned IdentityIndex = i *
Op.getValueType().getVectorMinNumElements();
5834 Op.getOperand(0).getValueType() != VT ||
5835 (IdentitySrc &&
Op.getOperand(0) != IdentitySrc) ||
5836 Op.getConstantOperandVal(1) != IdentityIndex) {
5840 assert((!IdentitySrc || IdentitySrc ==
Op.getOperand(0)) &&
5841 "Unexpected identity source vector for concat of extracts");
5842 IdentitySrc =
Op.getOperand(0);
5845 assert(IdentitySrc &&
"Failed to set source vector of extracts");
5860 EVT OpVT =
Op.getValueType();
5872 SVT = (SVT.
bitsLT(
Op.getValueType()) ?
Op.getValueType() : SVT);
5896 if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP))
5899 auto *
N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
5900 CSEMap.InsertNode(
N, IP);
5913 return getNode(Opcode,
DL, VT, N1, Flags);
5964 "STEP_VECTOR can only be used with scalable types");
5967 "Unexpected step operand");
5989 "Invalid FP cast!");
5993 "Vector element count mismatch!");
6011 "Invalid SIGN_EXTEND!");
6013 "SIGN_EXTEND result type type should be vector iff the operand "
6018 "Vector element count mismatch!");
6032 "Invalid ZERO_EXTEND!");
6034 "ZERO_EXTEND result type type should be vector iff the operand "
6039 "Vector element count mismatch!");
6070 "Invalid ANY_EXTEND!");
6072 "ANY_EXTEND result type type should be vector iff the operand "
6077 "Vector element count mismatch!");
6102 "Invalid TRUNCATE!");
6104 "TRUNCATE result type type should be vector iff the operand "
6109 "Vector element count mismatch!");
6132 assert(VT.
isVector() &&
"This DAG node is restricted to vector types.");
6134 "The input must be the same size or smaller than the result.");
6137 "The destination vector type must have fewer lanes than the input.");
6147 "BSWAP types must be a multiple of 16 bits!");
6161 "Cannot BITCAST between types of different sizes!");
6174 "Illegal SCALAR_TO_VECTOR node!");
6231 "Wrong operand type!");
6238 if (VT != MVT::Glue) {
6242 if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
6243 E->intersectFlagsWith(Flags);
6247 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
6249 createOperands(
N, Ops);
6250 CSEMap.InsertNode(
N, IP);
6252 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
6253 createOperands(
N, Ops);
6287 if (!C2.getBoolValue())
6291 if (!C2.getBoolValue())
6295 if (!C2.getBoolValue())
6299 if (!C2.getBoolValue())
6319 return std::nullopt;
6324 bool IsUndef1,
const APInt &C2,
6326 if (!(IsUndef1 || IsUndef2))
6334 return std::nullopt;
6344 auto *C2 = dyn_cast<ConstantSDNode>(N2);
6347 int64_t
Offset = C2->getSExtValue();
6365 assert(Ops.
size() == 2 &&
"Div/rem should have 2 operands");
6372 [](
SDValue V) { return V.isUndef() ||
6373 isNullConstant(V); });
6394 unsigned NumOps = Ops.
size();
6410 if (
auto *
C = dyn_cast<ConstantSDNode>(N1)) {
6411 const APInt &Val =
C->getAPIntValue();
6415 C->isTargetOpcode(),
C->isOpaque());
6422 C->isTargetOpcode(),
C->isOpaque());
6427 C->isTargetOpcode(),
C->isOpaque());
6429 C->isTargetOpcode(),
C->isOpaque());
6476 if (VT == MVT::f16 &&
C->getValueType(0) == MVT::i16)
6478 if (VT == MVT::f32 &&
C->getValueType(0) == MVT::i32)
6480 if (VT == MVT::f64 &&
C->getValueType(0) == MVT::i64)
6482 if (VT == MVT::f128 &&
C->getValueType(0) == MVT::i128)
6489 if (
auto *
C = dyn_cast<ConstantFPSDNode>(N1)) {
6543 return getConstant(V.bitcastToAPInt().getZExtValue(),
DL, VT);
6546 if (VT == MVT::i16 &&
C->getValueType(0) == MVT::f16)
6549 if (VT == MVT::i16 &&
C->getValueType(0) == MVT::bf16)
6552 if (VT == MVT::i32 &&
C->getValueType(0) == MVT::f32)
6555 if (VT == MVT::i64 &&
C->getValueType(0) == MVT::f64)
6556 return getConstant(V.bitcastToAPInt().getZExtValue(),
DL, VT);
6571 if (
auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
6572 if (
auto *C2 = dyn_cast<ConstantSDNode>(Ops[1])) {
6573 if (C1->isOpaque() || C2->isOpaque())
6576 std::optional<APInt> FoldAttempt =
6577 FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());
6583 "Can't fold vectors ops with scalar operands");
6604 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
6609 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
6610 auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
6617 if (BV1->getConstantRawBits(IsLE, EltBits, RawBits1, UndefElts1) &&
6618 BV2->getConstantRawBits(IsLE, EltBits, RawBits2, UndefElts2)) {
6622 Opcode, RawBits1[
I], UndefElts1[
I], RawBits2[
I], UndefElts2[
I]);
6633 BVEltVT = BV1->getOperand(0).getValueType();
6636 BVEltVT = BV2->getOperand(0).getValueType();
6642 DstBits, RawBits, DstUndefs,
6645 for (
unsigned I = 0, E = DstBits.
size();
I != E; ++
I) {
6663 ? Ops[0].getConstantOperandAPInt(0) * RHSVal
6664 : Ops[0].getConstantOperandAPInt(0) << RHSVal;
6669 auto IsScalarOrSameVectorSize = [NumElts](
const SDValue &
Op) {
6670 return !
Op.getValueType().isVector() ||
6671 Op.getValueType().getVectorElementCount() == NumElts;
6674 auto IsBuildVectorSplatVectorOrUndef = [](
const SDValue &
Op) {
6683 if (!
llvm::all_of(Ops, IsBuildVectorSplatVectorOrUndef) ||
6712 for (
unsigned I = 0;
I != NumVectorElts;
I++) {
6715 EVT InSVT =
Op.getValueType().getScalarType();
6737 !isa<ConstantSDNode>(ScalarOp) &&
6751 if (LegalSVT != SVT)
6752 ScalarResult =
getNode(ExtendCode,
DL, LegalSVT, ScalarResult);
6770 if (Ops.
size() != 2)
6781 if (N1CFP && N2CFP) {
6828 if (N1C && N1C->getValueAPF().isNegZero() && N2.
isUndef())
6857 ID.AddInteger(
A.value());
6860 if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP))
6864 newSDNode<AssertAlignSDNode>(
DL.getIROrder(),
DL.getDebugLoc(), VTs,
A);
6865 createOperands(
N, {Val});
6867 CSEMap.InsertNode(
N, IP);
6880 return getNode(Opcode,
DL, VT, N1, N2, Flags);
6894 if ((N1C && !N2C) || (N1CFP && !N2CFP))
6908 "Operand is DELETED_NODE!");
6912 auto *N1C = dyn_cast<ConstantSDNode>(N1);
6913 auto *N2C = dyn_cast<ConstantSDNode>(N2);
6924 N2.
getValueType() == MVT::Other &&
"Invalid token factor!");
6928 if (N1 == N2)
return N1;
6944 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
6946 N1.
getValueType() == VT &&
"Binary operator types must match!");
6949 if (N2CV && N2CV->
isZero())
6958 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
6960 N1.
getValueType() == VT &&
"Binary operator types must match!");
6963 if (N2CV && N2CV->
isZero())
6970 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
6972 N1.
getValueType() == VT &&
"Binary operator types must match!");
6977 const APInt &N2CImm = N2C->getAPIntValue();
6991 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
6993 N1.
getValueType() == VT &&
"Binary operator types must match!");
7006 "Types of operands of UCMP/SCMP must match");
7008 "Operands and return type of must both be scalars or vectors");
7012 "Result and operands must have the same number of elements");
7018 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
7020 N1.
getValueType() == VT &&
"Binary operator types must match!");
7024 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
7026 N1.
getValueType() == VT &&
"Binary operator types must match!");
7030 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
7032 N1.
getValueType() == VT &&
"Binary operator types must match!");
7038 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
7040 N1.
getValueType() == VT &&
"Binary operator types must match!");
7051 N1.
getValueType() == VT &&
"Binary operator types must match!");
7059 "Invalid FCOPYSIGN!");
7064 const APInt &ShiftImm = N2C->getAPIntValue();
7076 "Shift operators return type must be the same as their first arg");
7078 "Shifts only work on integers");
7080 "Vector shift amounts must be in the same as their first arg");
7087 "Invalid use of small shift amount with oversized value!");
7094 if (N2CV && N2CV->
isZero())
7101 N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
7102 "Invalid FP_ROUND!");
7107 EVT EVT = cast<VTSDNode>(N2)->getVT();
7110 "Cannot *_EXTEND_INREG FP types");
7112 "AssertSExt/AssertZExt type should be the vector element type "
7113 "rather than the vector type!");
7119 EVT EVT = cast<VTSDNode>(N2)->getVT();
7122 "Cannot *_EXTEND_INREG FP types");
7124 "SIGN_EXTEND_INREG type should be vector iff the operand "
7128 "Vector element counts must match in SIGN_EXTEND_INREG");
7130 if (
EVT == VT)
return N1;
7140 const APInt &Val = N1C->getAPIntValue();
7141 return SignExtendInReg(Val, VT);
7154 APInt Val =
C->getAPIntValue();
7155 Ops.
push_back(SignExtendInReg(Val, OpVT));
7173 "FP_TO_*INT_SAT type should be vector iff the operand type is "
7177 "Vector element counts must match in FP_TO_*INT_SAT");
7178 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
7179 "Type to saturate to must be a scalar.");
7186 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
7187 element type of the vector.");
7219 "BUILD_VECTOR used for scalable vectors");
7242 if (N1Op2C && N2C) {
7272 assert(N2C && (
unsigned)N2C->getZExtValue() < 2 &&
"Bad EXTRACT_ELEMENT!");
7276 "Wrong types for EXTRACT_ELEMENT!");
7287 unsigned Shift = ElementSize * N2C->getZExtValue();
7288 const APInt &Val = N1C->getAPIntValue();
7295 "Extract subvector VTs must be vectors!");
7297 "Extract subvector VTs must have the same element type!");
7299 "Cannot extract a scalable vector from a fixed length vector!");
7302 "Extract subvector must be from larger vector to smaller vector!");
7303 assert(N2C &&
"Extract subvector index must be a constant");
7307 "Extract subvector overflow!");
7308 assert(N2C->getAPIntValue().getBitWidth() ==
7310 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
7325 return N1.
getOperand(N2C->getZExtValue() / Factor);
7393 if (VT != MVT::Glue) {
7397 if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
7398 E->intersectFlagsWith(Flags);
7402 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
7404 createOperands(
N, Ops);
7405 CSEMap.InsertNode(
N, IP);
7407 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
7408 createOperands(
N, Ops);
7422 return getNode(Opcode,
DL, VT, N1, N2, N3, Flags);
7431 "Operand is DELETED_NODE!");
7442 if (N1CFP && N2CFP && N3CFP) {
7471 "SETCC operands must have the same type!");
7473 "SETCC type should be vector iff the operand type is vector!");
7476 "SETCC vector element counts must match!");
7496 if (cast<ConstantSDNode>(N3)->
isZero())
7526 "Dest and insert subvector source types must match!");
7528 "Insert subvector VTs must be vectors!");
7530 "Insert subvector VTs must have the same element type!");
7532 "Cannot insert a scalable vector into a fixed length vector!");
7535 "Insert subvector must be from smaller vector to larger vector!");
7536 assert(isa<ConstantSDNode>(N3) &&
7537 "Insert subvector index must be constant");
7541 "Insert subvector overflow!");
7544 "Constant index for INSERT_SUBVECTOR has an invalid size");
7562 case ISD::VP_TRUNCATE:
7563 case ISD::VP_SIGN_EXTEND:
7564 case ISD::VP_ZERO_EXTEND:
7573 assert(VT == VecVT &&
"Vector and result type don't match.");
7575 "All inputs must be vectors.");
7576 assert(VecVT == PassthruVT &&
"Vector and passthru types don't match.");
7578 "Vector and mask must have same number of elements.");
7591 if (VT != MVT::Glue) {
7595 if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
7596 E->intersectFlagsWith(Flags);
7600 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
7602 createOperands(
N, Ops);
7603 CSEMap.InsertNode(
N, IP);
7605 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
7606 createOperands(
N, Ops);
7617 SDValue Ops[] = { N1, N2, N3, N4 };
7624 SDValue Ops[] = { N1, N2, N3, N4, N5 };
7642 if (FI->getIndex() < 0)
7657 assert(
C->getAPIntValue().getBitWidth() == 8);
7662 return DAG.
getConstant(Val, dl, VT,
false, IsOpaque);
7668 assert(
Value.getValueType() == MVT::i8 &&
"memset with non-byte fill value?");
7684 if (VT !=
Value.getValueType())
7697 if (Slice.
Array ==
nullptr) {
7700 if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
7715 unsigned NumVTBytes = NumVTBits / 8;
7716 unsigned NumBytes = std::min(NumVTBytes,
unsigned(Slice.
Length));
7718 APInt Val(NumVTBits, 0);
7720 for (
unsigned i = 0; i != NumBytes; ++i)
7723 for (
unsigned i = 0; i != NumBytes; ++i)
7724 Val |= (
uint64_t)(
unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
7743 APInt(
Base.getValueSizeInBits().getFixedValue(),
7744 Offset.getKnownMinValue()));
7755 EVT BasePtrVT =
Ptr.getValueType();
7764 G = cast<GlobalAddressSDNode>(Src);
7765 else if (Src.getOpcode() ==
ISD::ADD &&
7768 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
7769 SrcDelta = Src.getConstantOperandVal(1);
7775 SrcDelta +
G->getOffset());
7791 assert(OutLoadChains.
size() &&
"Missing loads in memcpy inlining");
7792 assert(OutStoreChains.
size() &&
"Missing stores in memcpy inlining");
7794 for (
unsigned i =
From; i < To; ++i) {
7796 GluedLoadChains.
push_back(OutLoadChains[i]);
7803 for (
unsigned i =
From; i < To; ++i) {
7804 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
7806 ST->getBasePtr(), ST->getMemoryVT(),
7807 ST->getMemOperand());
7815 bool isVol,
bool AlwaysInline,
7831 std::vector<EVT> MemOps;
7832 bool DstAlignCanChange =
false;
7838 DstAlignCanChange =
true;
7840 if (!SrcAlign || Alignment > *SrcAlign)
7841 SrcAlign = Alignment;
7842 assert(SrcAlign &&
"SrcAlign must be set");
7846 bool isZeroConstant = CopyFromConstant && Slice.
Array ==
nullptr;
7848 const MemOp Op = isZeroConstant
7852 *SrcAlign, isVol, CopyFromConstant);
7858 if (DstAlignCanChange) {
7859 Type *Ty = MemOps[0].getTypeForEVT(
C);
7860 Align NewAlign =
DL.getABITypeAlign(Ty);
7866 if (!
TRI->hasStackRealignment(MF))
7867 while (NewAlign > Alignment &&
DL.exceedsNaturalStackAlignment(NewAlign))
7870 if (NewAlign > Alignment) {
7874 Alignment = NewAlign;
7882 const Value *SrcVal = dyn_cast_if_present<const Value *>(SrcPtrInfo.
V);
7892 unsigned NumMemOps = MemOps.
size();
7894 for (
unsigned i = 0; i != NumMemOps; ++i) {
7899 if (VTSize >
Size) {
7902 assert(i == NumMemOps-1 && i != 0);
7903 SrcOff -= VTSize -
Size;
7904 DstOff -= VTSize -
Size;
7907 if (CopyFromConstant &&
7915 if (SrcOff < Slice.
Length) {
7917 SubSlice.
move(SrcOff);
7920 SubSlice.
Array =
nullptr;
7922 SubSlice.
Length = VTSize;
7925 if (
Value.getNode()) {
7929 DstPtrInfo.
getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
7934 if (!Store.getNode()) {
7943 bool isDereferenceable =
7946 if (isDereferenceable)
7961 DstPtrInfo.
getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo);
7971 unsigned NumLdStInMemcpy = OutStoreChains.
size();
7973 if (NumLdStInMemcpy) {
7979 for (
unsigned i = 0; i < NumLdStInMemcpy; ++i) {
7985 if (NumLdStInMemcpy <= GluedLdStLimit) {
7987 NumLdStInMemcpy, OutLoadChains,
7990 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
7991 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
7992 unsigned GlueIter = 0;
7994 for (
unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
7995 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
7996 unsigned IndexTo = NumLdStInMemcpy - GlueIter;
7999 OutLoadChains, OutStoreChains);
8000 GlueIter += GluedLdStLimit;
8004 if (RemainingLdStInMemcpy) {
8006 RemainingLdStInMemcpy, OutLoadChains,
8018 bool isVol,
bool AlwaysInline,
8032 std::vector<EVT> MemOps;
8033 bool DstAlignCanChange =
false;
8039 DstAlignCanChange =
true;
8041 if (!SrcAlign || Alignment > *SrcAlign)
8042 SrcAlign = Alignment;
8043 assert(SrcAlign &&
"SrcAlign must be set");
8053 if (DstAlignCanChange) {
8054 Type *Ty = MemOps[0].getTypeForEVT(
C);
8055 Align NewAlign =
DL.getABITypeAlign(Ty);
8061 if (!
TRI->hasStackRealignment(MF))
8062 while (NewAlign > Alignment &&
DL.exceedsNaturalStackAlignment(NewAlign))
8065 if (NewAlign > Alignment) {
8069 Alignment = NewAlign;
8083 unsigned NumMemOps = MemOps.
size();
8084 for (
unsigned i = 0; i < NumMemOps; i++) {
8089 bool isDereferenceable =
8092 if (isDereferenceable)
8098 SrcPtrInfo.
getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo);
8105 for (
unsigned i = 0; i < NumMemOps; i++) {
8111 Chain, dl, LoadValues[i],
8113 DstPtrInfo.
getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
8153 std::vector<EVT> MemOps;
8154 bool DstAlignCanChange =
false;
8160 DstAlignCanChange =
true;
8166 MemOp::Set(
Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),
8170 if (DstAlignCanChange) {
8173 Align NewAlign =
DL.getABITypeAlign(Ty);
8179 if (!
TRI->hasStackRealignment(MF))
8180 while (NewAlign > Alignment &&
DL.exceedsNaturalStackAlignment(NewAlign))
8183 if (NewAlign > Alignment) {
8187 Alignment = NewAlign;
8193 unsigned NumMemOps = MemOps.size();
8196 EVT LargestVT = MemOps[0];
8197 for (
unsigned i = 1; i < NumMemOps; i++)
8198 if (MemOps[i].bitsGT(LargestVT))
8199 LargestVT = MemOps[i];
8206 for (
unsigned i = 0; i < NumMemOps; i++) {
8209 if (VTSize >
Size) {
8212 assert(i == NumMemOps-1 && i != 0);
8213 DstOff -= VTSize -
Size;
8220 if (VT.
bitsLT(LargestVT)) {
8241 assert(
Value.getValueType() == VT &&
"Value with wrong type.");
8268 Align Alignment,
bool isVol,
bool AlwaysInline,
const CallInst *CI,
8276 if (ConstantSize->
isZero())
8280 *
this, dl, Chain, Dst, Src, ConstantSize->
getZExtValue(), Alignment,
8281 isVol,
false, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
8282 if (Result.getNode())
8290 *
this, dl, Chain, Dst, Src,
Size, Alignment, isVol, AlwaysInline,
8291 DstPtrInfo, SrcPtrInfo);
8292 if (Result.getNode())
8299 assert(ConstantSize &&
"AlwaysInline requires a constant size!");
8301 *
this, dl, Chain, Dst, Src, ConstantSize->
getZExtValue(), Alignment,
8302 isVol,
true, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
8318 Entry.Node = Dst; Args.push_back(Entry);
8319 Entry.Node = Src; Args.push_back(Entry);
8322 Entry.Node =
Size; Args.push_back(Entry);
8325 bool IsTailCall =
false;
8326 if (OverrideTailCall.has_value()) {
8327 IsTailCall = *OverrideTailCall;
8329 bool LowersToMemcpy =
8334 ReturnsFirstArg && LowersToMemcpy);
8340 Dst.getValueType().getTypeForEVT(*
getContext()),
8347 std::pair<SDValue,SDValue> CallResult = TLI->
LowerCallTo(CLI);
8348 return CallResult.second;
8353 Type *SizeTy,
unsigned ElemSz,
8362 Args.push_back(Entry);
8365 Args.push_back(Entry);
8369 Args.push_back(Entry);
8373 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8387 std::pair<SDValue, SDValue> CallResult = TLI->
LowerCallTo(CLI);
8388 return CallResult.second;
8394 std::optional<bool> OverrideTailCall,
8403 if (ConstantSize->
isZero())
8407 *
this, dl, Chain, Dst, Src, ConstantSize->
getZExtValue(), Alignment,
8408 isVol,
false, DstPtrInfo, SrcPtrInfo, AAInfo);
8409 if (Result.getNode())
8418 Alignment, isVol, DstPtrInfo, SrcPtrInfo);
8419 if (Result.getNode())
8433 Entry.Node = Dst; Args.push_back(Entry);
8434 Entry.Node = Src; Args.push_back(Entry);
8437 Entry.Node =
Size; Args.push_back(Entry);
8441 bool IsTailCall =
false;
8442 if (OverrideTailCall.has_value()) {
8443 IsTailCall = *OverrideTailCall;
8445 bool LowersToMemmove =
8450 ReturnsFirstArg && LowersToMemmove);
8456 Dst.getValueType().getTypeForEVT(*
getContext()),
8463 std::pair<SDValue,SDValue> CallResult = TLI->
LowerCallTo(CLI);
8464 return CallResult.second;
8469 Type *SizeTy,
unsigned ElemSz,
8478 Args.push_back(Entry);
8481 Args.push_back(Entry);
8485 Args.push_back(Entry);
8489 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8503 std::pair<SDValue, SDValue> CallResult = TLI->
LowerCallTo(CLI);
8504 return CallResult.second;
8509 bool isVol,
bool AlwaysInline,
8518 if (ConstantSize->
isZero())
8523 isVol,
false, DstPtrInfo, AAInfo);
8525 if (Result.getNode())
8533 *
this, dl, Chain, Dst, Src,
Size, Alignment, isVol, AlwaysInline, DstPtrInfo);
8534 if (Result.getNode())
8541 assert(ConstantSize &&
"AlwaysInline requires a constant size!");
8544 isVol,
true, DstPtrInfo, AAInfo);
8546 "getMemsetStores must return a valid sequence when AlwaysInline");
8563 const auto CreateEntry = [](
SDValue Node,
Type *Ty) {
8575 Args.push_back(CreateEntry(
Size,
DL.getIntPtrType(Ctx)));
8582 Args.push_back(CreateEntry(Src, Src.getValueType().getTypeForEVT(Ctx)));
8583 Args.push_back(CreateEntry(
Size,
DL.getIntPtrType(Ctx)));
8585 Dst.getValueType().getTypeForEVT(Ctx),
8590 bool LowersToMemset =
8601 std::pair<SDValue, SDValue> CallResult = TLI->
LowerCallTo(CLI);
8602 return CallResult.second;
8607 Type *SizeTy,
unsigned ElemSz,
8615 Args.push_back(Entry);
8619 Args.push_back(Entry);
8623 Args.push_back(Entry);
8627 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8641 std::pair<SDValue, SDValue> CallResult = TLI->
LowerCallTo(CLI);
8642 return CallResult.second;
8654 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
8655 cast<AtomicSDNode>(E)->refineAlignment(MMO);
8660 VTList, MemVT, MMO);
8661 createOperands(
N, Ops);
8663 CSEMap.InsertNode(
N, IP);
8677 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8702 "Invalid Atomic Op");
8709 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8719 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8724 if (Ops.
size() == 1)
8739 if (
Size.hasValue() && !
Size.getValue())
8756 (Opcode <= (
unsigned)std::numeric_limits<int>::max() &&
8758 "Opcode is not a memory-accessing opcode!");
8762 if (VTList.
VTs[VTList.
NumVTs-1] != MVT::Glue) {
8765 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
8766 Opcode, dl.
getIROrder(), VTList, MemVT, MMO));
8771 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
8772 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
8777 VTList, MemVT, MMO);
8778 createOperands(
N, Ops);
8780 CSEMap.InsertNode(
N, IP);
8783 VTList, MemVT, MMO);
8784 createOperands(
N, Ops);
8793 SDValue Chain,
int FrameIndex,
8805 ID.AddInteger(FrameIndex);
8809 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
8814 createOperands(
N, Ops);
8815 CSEMap.InsertNode(
N, IP);
8833 if (
SDNode *E = FindNodeOrInsertPos(
ID, Dl, IP))
8836 auto *
N = newSDNode<PseudoProbeSDNode>(
8838 createOperands(
N, Ops);
8839 CSEMap.InsertNode(
N, IP);
8860 !isa<ConstantSDNode>(
Ptr.getOperand(1)) ||
8861 !isa<FrameIndexSDNode>(
Ptr.getOperand(0)))
8864 int FI = cast<FrameIndexSDNode>(
Ptr.getOperand(0))->getIndex();
8867 Offset + cast<ConstantSDNode>(
Ptr.getOperand(1))->getSExtValue());
8878 if (
ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
8893 "Invalid chain type");
8905 Alignment, AAInfo, Ranges);
8916 assert(VT == MemVT &&
"Non-extending load from different memory type!");
8920 "Should only be an extending load, not truncating!");
8922 "Cannot convert from FP to Int or Int -> FP!");
8924 "Cannot use an ext load to convert to or from a vector!");
8927 "Cannot use an ext load to change the number of vector elements!");
8939 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
8940 dl.
getIROrder(), VTs, AM, ExtType, MemVT, MMO));
8944 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
8945 cast<LoadSDNode>(E)->refineAlignment(MMO);
8949 ExtType, MemVT, MMO);
8950 createOperands(
N, Ops);
8952 CSEMap.InsertNode(
N, IP);
8966 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
8984 MemVT, Alignment, MMOFlags, AAInfo);
8999 assert(LD->getOffset().isUndef() &&
"Load is already a indexed load!");
9002 LD->getMemOperand()->getFlags() &
9005 LD->getChain(),
Base,
Offset, LD->getPointerInfo(),
9006 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo());
9032 "Invalid chain type");
9040 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
9045 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9046 cast<StoreSDNode>(E)->refineAlignment(MMO);
9051 createOperands(
N, Ops);
9053 CSEMap.InsertNode(
N, IP);
9066 "Invalid chain type");
9087 "Invalid chain type");
9092 "Should only be a truncating store, not extending!");
9094 "Can't do FP-INT conversion!");
9096 "Cannot use trunc store to convert to or from a vector!");
9099 "Cannot use trunc store to change the number of vector elements!");
9107 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
9112 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9113 cast<StoreSDNode>(E)->refineAlignment(MMO);
9118 createOperands(
N, Ops);
9120 CSEMap.InsertNode(
N, IP);
9131 assert(ST->getOffset().isUndef() &&
"Store is already a indexed store!");
9136 ID.AddInteger(ST->getMemoryVT().getRawBits());
9137 ID.AddInteger(ST->getRawSubclassData());
9138 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
9139 ID.AddInteger(ST->getMemOperand()->getFlags());
9141 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
9145 ST->isTruncatingStore(), ST->getMemoryVT(),
9146 ST->getMemOperand());
9147 createOperands(
N, Ops);
9149 CSEMap.InsertNode(
N, IP);
9161 const MDNode *Ranges,
bool IsExpanding) {
9174 Alignment, AAInfo, Ranges);
9175 return getLoadVP(AM, ExtType, VT, dl, Chain,
Ptr,
Offset, Mask, EVL, MemVT,
9194 ID.AddInteger(getSyntheticNodeSubclassData<VPLoadSDNode>(
9195 dl.
getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
9199 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9200 cast<VPLoadSDNode>(E)->refineAlignment(MMO);
9204 ExtType, IsExpanding, MemVT, MMO);
9205 createOperands(
N, Ops);
9207 CSEMap.InsertNode(
N, IP);
9223 Mask, EVL, PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges,
9232 Mask, EVL, VT, MMO, IsExpanding);
9241 const AAMDNodes &AAInfo,
bool IsExpanding) {
9244 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo,
nullptr,
9254 EVL, MemVT, MMO, IsExpanding);
9260 auto *LD = cast<VPLoadSDNode>(OrigLoad);
9261 assert(LD->getOffset().isUndef() &&
"Load is already a indexed load!");
9264 LD->getMemOperand()->getFlags() &
9268 LD->getVectorLength(), LD->getPointerInfo(),
9269 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo(),
9270 nullptr, LD->isExpandingLoad());
9277 bool IsCompressing) {
9287 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
9288 dl.
getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9292 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9293 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
9297 IsTruncating, IsCompressing, MemVT, MMO);
9298 createOperands(
N, Ops);
9300 CSEMap.InsertNode(
N, IP);
9313 bool IsCompressing) {
9334 bool IsCompressing) {
9341 false, IsCompressing);
9344 "Should only be a truncating store, not extending!");
9347 "Cannot use trunc store to convert to or from a vector!");
9350 "Cannot use trunc store to change the number of vector elements!");
9354 SDValue Ops[] = {Chain, Val,
Ptr, Undef, Mask, EVL};
9358 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
9363 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9364 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
9370 createOperands(
N, Ops);
9372 CSEMap.InsertNode(
N, IP);
9382 auto *ST = cast<VPStoreSDNode>(OrigStore);
9383 assert(ST->getOffset().isUndef() &&
"Store is already an indexed store!");
9385 SDValue Ops[] = {ST->getChain(), ST->getValue(),
Base,
9386 Offset, ST->getMask(), ST->getVectorLength()};
9389 ID.AddInteger(ST->getMemoryVT().getRawBits());
9390 ID.AddInteger(ST->getRawSubclassData());
9391 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
9392 ID.AddInteger(ST->getMemOperand()->getFlags());
9394 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
9397 auto *
N = newSDNode<VPStoreSDNode>(
9399 ST->isCompressingStore(), ST->getMemoryVT(), ST->getMemOperand());
9400 createOperands(
N, Ops);
9402 CSEMap.InsertNode(
N, IP);
9422 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
9423 DL.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
9427 if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
9428 cast<VPStridedLoadSDNode>(E)->refineAlignment(MMO);
9433 newSDNode<VPStridedLoadSDNode>(
DL.getIROrder(),
DL.getDebugLoc(), VTs, AM,
9434 ExtType, IsExpanding, MemVT, MMO);
9435 createOperands(
N, Ops);
9436 CSEMap.InsertNode(
N, IP);
9450 Undef, Stride, Mask, EVL, VT, MMO, IsExpanding);
9459 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
9468 bool IsTruncating,
bool IsCompressing) {
9478 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9479 DL.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9482 if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
9483 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9486 auto *
N = newSDNode<VPStridedStoreSDNode>(
DL.getIROrder(),
DL.getDebugLoc(),
9487 VTs, AM, IsTruncating,
9488 IsCompressing, MemVT, MMO);
9489 createOperands(
N, Ops);
9491 CSEMap.InsertNode(
N, IP);
9503 bool IsCompressing) {
9510 false, IsCompressing);
9513 "Should only be a truncating store, not extending!");
9516 "Cannot use trunc store to convert to or from a vector!");
9519 "Cannot use trunc store to change the number of vector elements!");
9523 SDValue Ops[] = {Chain, Val,
Ptr, Undef, Stride, Mask, EVL};
9527 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9531 if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
9532 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9535 auto *
N = newSDNode<VPStridedStoreSDNode>(
DL.getIROrder(),
DL.getDebugLoc(),
9537 IsCompressing, SVT, MMO);
9538 createOperands(
N, Ops);
9540 CSEMap.InsertNode(
N, IP);
9550 assert(Ops.
size() == 6 &&
"Incompatible number of operands");
9555 ID.AddInteger(getSyntheticNodeSubclassData<VPGatherSDNode>(
9560 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9561 cast<VPGatherSDNode>(E)->refineAlignment(MMO);
9566 VT, MMO, IndexType);
9567 createOperands(
N, Ops);
9569 assert(
N->getMask().getValueType().getVectorElementCount() ==
9570 N->getValueType(0).getVectorElementCount() &&
9571 "Vector width mismatch between mask and data");
9572 assert(
N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9573 N->getValueType(0).getVectorElementCount().isScalable() &&
9574 "Scalable flags of index and data do not match");
9576 N->getIndex().getValueType().getVectorElementCount(),
9577 N->getValueType(0).getVectorElementCount()) &&
9578 "Vector width mismatch between index and data");
9579 assert(isa<ConstantSDNode>(
N->getScale()) &&
9580 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9581 "Scale should be a constant power of 2");
9583 CSEMap.InsertNode(
N, IP);
9594 assert(Ops.
size() == 7 &&
"Incompatible number of operands");
9599 ID.AddInteger(getSyntheticNodeSubclassData<VPScatterSDNode>(
9604 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9605 cast<VPScatterSDNode>(E)->refineAlignment(MMO);
9609 VT, MMO, IndexType);
9610 createOperands(
N, Ops);
9612 assert(
N->getMask().getValueType().getVectorElementCount() ==
9613 N->getValue().getValueType().getVectorElementCount() &&
9614 "Vector width mismatch between mask and data");
9616 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9617 N->getValue().getValueType().getVectorElementCount().isScalable() &&
9618 "Scalable flags of index and data do not match");
9620 N->getIndex().getValueType().getVectorElementCount(),
9621 N->getValue().getValueType().getVectorElementCount()) &&
9622 "Vector width mismatch between index and data");
9623 assert(isa<ConstantSDNode>(
N->getScale()) &&
9624 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9625 "Scale should be a constant power of 2");
9627 CSEMap.InsertNode(
N, IP);
9642 "Unindexed masked load with an offset!");
9649 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
9650 dl.
getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));
9654 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9655 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
9659 AM, ExtTy, isExpanding, MemVT, MMO);
9660 createOperands(
N, Ops);
9662 CSEMap.InsertNode(
N, IP);
9673 assert(LD->getOffset().isUndef() &&
"Masked load is already a indexed load!");
9675 Offset, LD->getMask(), LD->getPassThru(),
9676 LD->getMemoryVT(), LD->getMemOperand(), AM,
9677 LD->getExtensionType(), LD->isExpandingLoad());
9685 bool IsCompressing) {
9687 "Invalid chain type");
9690 "Unindexed masked store with an offset!");
9697 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
9698 dl.
getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9702 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9703 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
9708 IsTruncating, IsCompressing, MemVT, MMO);
9709 createOperands(
N, Ops);
9711 CSEMap.InsertNode(
N, IP);
9722 assert(ST->getOffset().isUndef() &&
9723 "Masked store is already a indexed store!");
9725 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
9726 AM, ST->isTruncatingStore(), ST->isCompressingStore());
9734 assert(Ops.
size() == 6 &&
"Incompatible number of operands");
9739 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
9740 dl.
getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));
9744 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9745 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
9750 VTs, MemVT, MMO, IndexType, ExtTy);
9751 createOperands(
N, Ops);
9753 assert(
N->getPassThru().getValueType() ==
N->getValueType(0) &&
9754 "Incompatible type of the PassThru value in MaskedGatherSDNode");
9755 assert(
N->getMask().getValueType().getVectorElementCount() ==
9756 N->getValueType(0).getVectorElementCount() &&
9757 "Vector width mismatch between mask and data");
9758 assert(
N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9759 N->getValueType(0).getVectorElementCount().isScalable() &&
9760 "Scalable flags of index and data do not match");
9762 N->getIndex().getValueType().getVectorElementCount(),
9763 N->getValueType(0).getVectorElementCount()) &&
9764 "Vector width mismatch between index and data");
9765 assert(isa<ConstantSDNode>(
N->getScale()) &&
9766 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9767 "Scale should be a constant power of 2");
9769 CSEMap.InsertNode(
N, IP);
9781 assert(Ops.
size() == 6 &&
"Incompatible number of operands");
9786 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
9787 dl.
getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));
9791 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9792 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
9797 VTs, MemVT, MMO, IndexType, IsTrunc);
9798 createOperands(
N, Ops);
9800 assert(
N->getMask().getValueType().getVectorElementCount() ==
9801 N->getValue().getValueType().getVectorElementCount() &&
9802 "Vector width mismatch between mask and data");
9804 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9805 N->getValue().getValueType().getVectorElementCount().isScalable() &&
9806 "Scalable flags of index and data do not match");
9808 N->getIndex().getValueType().getVectorElementCount(),
9809 N->getValue().getValueType().getVectorElementCount()) &&
9810 "Vector width mismatch between index and data");
9811 assert(isa<ConstantSDNode>(
N->getScale()) &&
9812 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9813 "Scale should be a constant power of 2");
9815 CSEMap.InsertNode(
N, IP);
9826 assert(Ops.
size() == 7 &&
"Incompatible number of operands");
9831 ID.AddInteger(getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
9832 dl.
getIROrder(), VTs, MemVT, MMO, IndexType));
9836 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP)) {
9837 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
9842 VTs, MemVT, MMO, IndexType);
9843 createOperands(
N, Ops);
9845 assert(
N->getMask().getValueType().getVectorElementCount() ==
9846 N->getIndex().getValueType().getVectorElementCount() &&
9847 "Vector width mismatch between mask and data");
9848 assert(isa<ConstantSDNode>(
N->getScale()) &&
9849 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9850 "Scale should be a constant power of 2");
9851 assert(
N->getInc().getValueType().isInteger() &&
"Non integer update value");
9853 CSEMap.InsertNode(
N, IP);
9868 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9873 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
9878 createOperands(
N, Ops);
9880 CSEMap.InsertNode(
N, IP);
9895 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9900 if (
SDNode *E = FindNodeOrInsertPos(
ID, dl, IP))
9905 createOperands(
N, Ops);
9907 CSEMap.InsertNode(
N, IP);
9953 return !Val || Val->getAPIntValue().uge(
X.getScalarValueSizeInBits());
9959 if (
X.getValueType().getScalarType() == MVT::i1)
9972 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
9974 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
9977 if (Flags.hasNoNaNs() && (HasNan ||
X.isUndef() ||
Y.isUndef()))
9980 if (Flags.hasNoInfs() && (HasInf ||
X.isUndef() ||
Y.isUndef()))
10003 if (Opcode ==
ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
10018 switch (Ops.
size()) {
10019 case 0:
return getNode(Opcode,
DL, VT);
10020 case 1:
return getNode(Opcode,
DL, VT,
static_cast<const SDValue>(Ops[0]));
10021 case 2:
return getNode(Opcode,
DL, VT, Ops[0], Ops[1]);
10022 case 3:
return getNode(Opcode,
DL, VT, Ops[0], Ops[1], Ops[2]);
10029 return getNode(Opcode,
DL, VT, NewOps);
10037 return getNode(Opcode,
DL, VT, Ops, Flags);
10042 unsigned NumOps = Ops.
size();
10044 case 0:
return getNode(Opcode,
DL, VT);
10045 case 1:
return getNode(Opcode,
DL, VT, Ops[0], Flags);
10046 case 2:
return getNode(Opcode,
DL, VT, Ops[0], Ops[1], Flags);
10047 case 3:
return getNode(Opcode,
DL, VT, Ops[0], Ops[1], Ops[2], Flags);
10052 for (
const auto &
Op : Ops)
10054 "Operand is DELETED_NODE!");
10069 assert(NumOps == 5 &&
"SELECT_CC takes 5 operands!");
10071 "LHS and RHS of condition must have same type!");
10073 "True and False arms of SelectCC must have same type!");
10075 "select_cc node must be of same type as true and false value!");
10079 "Expected select_cc with vector result to have the same sized "
10080 "comparison type!");
10083 assert(NumOps == 5 &&
"BR_CC takes 5 operands!");
10085 "LHS/RHS of comparison should match types!");
10091 Opcode = ISD::VP_XOR;
10096 Opcode = ISD::VP_AND;
10098 case ISD::VP_REDUCE_MUL:
10101 Opcode = ISD::VP_REDUCE_AND;
10103 case ISD::VP_REDUCE_ADD:
10106 Opcode = ISD::VP_REDUCE_XOR;
10108 case ISD::VP_REDUCE_SMAX:
10109 case ISD::VP_REDUCE_UMIN:
10113 Opcode = ISD::VP_REDUCE_AND;
10115 case ISD::VP_REDUCE_SMIN:
10116 case ISD::VP_REDUCE_UMAX:
10120 Opcode = ISD::VP_REDUCE_OR;
10128 if (VT != MVT::Glue) {
10131 void *IP =
nullptr;
10133 if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP))
10136 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
10137 createOperands(
N, Ops);
10139 CSEMap.InsertNode(
N, IP);
10141 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
10142 createOperands(
N, Ops);
10145 N->setFlags(Flags);
10162 return getNode(Opcode,
DL, VTList, Ops, Flags);
10168 return getNode(Opcode,
DL, VTList.
VTs[0], Ops, Flags);
10171 for (
const auto &
Op : Ops)
10173 "Operand is DELETED_NODE!");
10182 "Invalid add/sub overflow op!");
10184 Ops[0].getValueType() == Ops[1].getValueType() &&
10185 Ops[0].getValueType() == VTList.
VTs[0] &&
10186 "Binary operator types must match!");
10187 SDValue N1 = Ops[0], N2 = Ops[1];
10193 if (N2CV && N2CV->
isZero()) {
10225 "Invalid add/sub overflow op!");
10227 Ops[0].getValueType() == Ops[1].getValueType() &&
10228 Ops[0].getValueType() == VTList.
VTs[0] &&
10229 Ops[2].getValueType() == VTList.
VTs[1] &&
10230 "Binary operator types must match!");
10236 VTList.
VTs[0] == Ops[0].getValueType() &&
10237 VTList.
VTs[0] == Ops[1].getValueType() &&
10238 "Binary operator types must match!");
10244 unsigned OutWidth = Width * 2;
10248 Val = Val.
sext(OutWidth);
10249 Mul =
Mul.sext(OutWidth);
10251 Val = Val.
zext(OutWidth);
10252 Mul =
Mul.zext(OutWidth);
10266 VTList.
VTs[0] == Ops[0].getValueType() &&
"frexp type mismatch");
10282 "Invalid STRICT_FP_EXTEND!");
10284 Ops[1].getValueType().isFloatingPoint() &&
"Invalid FP cast!");
10286 "STRICT_FP_EXTEND result type should be vector iff the operand "
10287 "type is vector!");
10290 Ops[1].getValueType().getVectorElementCount()) &&
10291 "Vector element count mismatch!");
10293 "Invalid fpext node, dst <= src!");
10296 assert(VTList.
NumVTs == 2 && Ops.
size() == 3 &&
"Invalid STRICT_FP_ROUND!");
10298 "STRICT_FP_ROUND result type should be vector iff the operand "
10299 "type is vector!");
10302 Ops[1].getValueType().getVectorElementCount()) &&
10303 "Vector element count mismatch!");
10305 Ops[1].getValueType().isFloatingPoint() &&
10306 VTList.
VTs[0].
bitsLT(Ops[1].getValueType()) &&
10307 isa<ConstantSDNode>(Ops[2]) &&
10308 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
10309 "Invalid STRICT_FP_ROUND!");
10319 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
10320 return getNode(Opcode,
DL, VT, N1, N2, N3.getOperand(0));
10321 else if (N3.getOpcode() ==
ISD::AND)
10322 if (
ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
10326 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
10327 return getNode(Opcode,
DL, VT, N1, N2, N3.getOperand(0));
10335 if (VTList.
VTs[VTList.
NumVTs-1] != MVT::Glue) {
10338 void *IP =
nullptr;
10339 if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP))
10342 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTList);
10343 createOperands(
N, Ops);
10344 CSEMap.InsertNode(
N, IP);
10346 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTList);
10347 createOperands(
N, Ops);
10350 N->setFlags(Flags);
10359 return getNode(Opcode,
DL, VTList, std::nullopt);
10365 return getNode(Opcode,
DL, VTList, Ops);
10371 return getNode(Opcode,
DL, VTList, Ops);
10376 SDValue Ops[] = { N1, N2, N3 };
10377 return getNode(Opcode,
DL, VTList, Ops);
10382 SDValue Ops[] = { N1, N2, N3, N4 };
10383 return getNode(Opcode,
DL, VTList, Ops);
10389 SDValue Ops[] = { N1, N2, N3, N4, N5 };
10390 return getNode(Opcode,
DL, VTList, Ops);
10394 return makeVTList(SDNode::getValueTypeList(VT), 1);
10403 void *IP =
nullptr;
10409 Result =
new (Allocator)
SDVTListNode(
ID.Intern(Allocator), Array, 2);
10410 VTListMap.InsertNode(Result, IP);
10412 return Result->getSDVTList();
10422 void *IP =
nullptr;
10429 Result =
new (Allocator)
SDVTListNode(
ID.Intern(Allocator), Array, 3);
10430 VTListMap.InsertNode(Result, IP);
10432 return Result->getSDVTList();
10443 void *IP =
nullptr;
10451 Result =
new (Allocator)
SDVTListNode(
ID.Intern(Allocator), Array, 4);
10452 VTListMap.InsertNode(Result, IP);
10454 return Result->getSDVTList();
10458 unsigned NumVTs = VTs.
size();
10460 ID.AddInteger(NumVTs);
10461 for (
unsigned index = 0; index < NumVTs; index++) {
10462 ID.AddInteger(VTs[index].getRawBits());
10465 void *IP =
nullptr;
10470 Result =
new (Allocator)
SDVTListNode(
ID.Intern(Allocator), Array, NumVTs);
10471 VTListMap.InsertNode(Result, IP);
10473 return Result->getSDVTList();
10484 assert(
N->getNumOperands() == 1 &&
"Update with wrong number of operands");
10487 if (
Op ==
N->getOperand(0))
return N;
10490 void *InsertPos =
nullptr;
10491 if (
SDNode *Existing = FindModifiedNodeSlot(
N,
Op, InsertPos))
10496 if (!RemoveNodeFromCSEMaps(
N))
10497 InsertPos =
nullptr;
10500 N->OperandList[0].set(
Op);
10504 if (InsertPos) CSEMap.InsertNode(
N, InsertPos);
10509 assert(
N->getNumOperands() == 2 &&
"Update with wrong number of operands");
10512 if (Op1 ==
N->getOperand(0) && Op2 ==
N->getOperand(1))
10516 void *InsertPos =
nullptr;
10517 if (
SDNode *Existing = FindModifiedNodeSlot(
N, Op1, Op2, InsertPos))
10522 if (!RemoveNodeFromCSEMaps(
N))
10523 InsertPos =
nullptr;
10526 if (
N->OperandList[0] != Op1)
10527 N->OperandList[0].set(Op1);
10528 if (
N->OperandList[1] != Op2)
10529 N->OperandList[1].set(Op2);
10533 if (InsertPos) CSEMap.InsertNode(
N, InsertPos);
10539 SDValue Ops[] = { Op1, Op2, Op3 };
10546 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
10553 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
10559 unsigned NumOps = Ops.
size();
10560 assert(
N->getNumOperands() == NumOps &&
10561 "Update with wrong number of operands");
10564 if (std::equal(Ops.
begin(), Ops.
end(),
N->op_begin()))
10568 void *InsertPos =
nullptr;
10569 if (
SDNode *Existing = FindModifiedNodeSlot(
N, Ops, InsertPos))
10574 if (!RemoveNodeFromCSEMaps(
N))
10575 InsertPos =
nullptr;
10578 for (
unsigned i = 0; i != NumOps; ++i)
10579 if (
N->OperandList[i] != Ops[i])
10580 N->OperandList[i].set(Ops[i]);
10584 if (InsertPos) CSEMap.InsertNode(
N, InsertPos);
10601 if (NewMemRefs.
empty()) {
10607 if (NewMemRefs.
size() == 1) {
10608 N->MemRefs = NewMemRefs[0];
10614 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.
size());
10616 N->MemRefs = MemRefsBuffer;
10617 N->NumMemRefs =
static_cast<int>(NewMemRefs.
size());
10640 SDValue Ops[] = { Op1, Op2 };
10648 SDValue Ops[] = { Op1, Op2, Op3 };
10681 SDValue Ops[] = { Op1, Op2 };
10689 New->setNodeId(-1);
10709 unsigned Order = std::min(
N->getIROrder(), OLoc.
getIROrder());
10710 N->setIROrder(Order);
10733 void *IP =
nullptr;
10734 if (VTs.
VTs[VTs.
NumVTs-1] != MVT::Glue) {
10738 return UpdateSDLocOnMergeSDNode(ON,
SDLoc(
N));
10741 if (!RemoveNodeFromCSEMaps(
N))
10746 N->ValueList = VTs.
VTs;
10756 if (Used->use_empty())
10757 DeadNodeSet.
insert(Used);
10762 MN->clearMemRefs();
10766 createOperands(
N, Ops);
10770 if (!DeadNodeSet.
empty()) {
10772 for (
SDNode *
N : DeadNodeSet)
10773 if (
N->use_empty())
10779 CSEMap.InsertNode(
N, IP);
10784 unsigned OrigOpc = Node->getOpcode();
10789#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
10790 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
10791#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
10792 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
10793#include "llvm/IR/ConstrainedOps.def"
10796 assert(Node->getNumValues() == 2 &&
"Unexpected number of results!");
10799 SDValue InputChain = Node->getOperand(0);
10804 for (
unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
10847 SDValue Ops[] = { Op1, Op2 };
10855 SDValue Ops[] = { Op1, Op2, Op3 };
10869 SDValue Ops[] = { Op1, Op2 };
10877 SDValue Ops[] = { Op1, Op2, Op3 };
10892 SDValue Ops[] = { Op1, Op2 };
10901 SDValue Ops[] = { Op1, Op2, Op3 };
10922 bool DoCSE = VTs.
VTs[VTs.
NumVTs-1] != MVT::Glue;
10924 void *IP =
nullptr;
10930 if (
SDNode *E = FindNodeOrInsertPos(
ID,
DL, IP)) {
10931 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E,
DL));
10936 N = newSDNode<MachineSDNode>(~Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
10937 createOperands(
N, Ops);
10940 CSEMap.InsertNode(
N, IP);
10953 VT, Operand, SRIdxVal);
10963 VT, Operand, Subreg, SRIdxVal);
10980 if (VTList.
VTs[VTList.
NumVTs - 1] != MVT::Glue) {
10983 void *IP =
nullptr;
10985 E->intersectFlagsWith(Flags);
10995 if (VTList.
VTs[VTList.
NumVTs - 1] != MVT::Glue) {
10998 void *IP =
nullptr;
10999 if (FindNodeOrInsertPos(
ID,
SDLoc(), IP))
11009 SDNode *
N,
unsigned R,
bool IsIndirect,
11011 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(
DL) &&
11012 "Expected inlined-at fields to agree");
11015 {}, IsIndirect,
DL, O,
11024 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(
DL) &&
11025 "Expected inlined-at fields to agree");
11038 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(
DL) &&
11039 "Expected inlined-at fields to agree");
11050 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(
DL) &&
11051 "Expected inlined-at fields to agree");
11054 Dependencies, IsIndirect,
DL, O,
11060 unsigned VReg,
bool IsIndirect,
11062 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(
DL) &&
11063 "Expected inlined-at fields to agree");
11066 {}, IsIndirect,
DL, O,
11074 unsigned O,
bool IsVariadic) {
11075 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(
DL) &&
11076 "Expected inlined-at fields to agree");
11079 DL, O, IsVariadic);
11083 unsigned OffsetInBits,
unsigned SizeInBits,
11084 bool InvalidateDbg) {
11087 assert(FromNode && ToNode &&
"Can't modify dbg values");
11092 if (
From == To || FromNode == ToNode)
11104 if (Dbg->isInvalidated())
11111 bool Changed =
false;
11112 auto NewLocOps = Dbg->copyLocationOps();
11114 NewLocOps.begin(), NewLocOps.end(),
11116 bool Match = Op == FromLocOp;
11126 auto *Expr = Dbg->getExpression();
11132 if (
auto FI = Expr->getFragmentInfo())
11133 if (OffsetInBits + SizeInBits > FI->SizeInBits)
11142 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
11145 Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),
11146 Dbg->getDebugLoc(), std::max(ToNode->
getIROrder(), Dbg->getOrder()),
11147 Dbg->isVariadic());
11150 if (InvalidateDbg) {
11152 Dbg->setIsInvalidated();
11153 Dbg->setIsEmitted();
11159 "Transferred DbgValues should depend on the new SDNode");
11165 if (!
N.getHasDebugValue())
11170 if (DV->isInvalidated())
11172 switch (
N.getOpcode()) {
11178 if (!isa<ConstantSDNode>(N0)) {
11179 bool RHSConstant = isa<ConstantSDNode>(N1);
11182 Offset =
N.getConstantOperandVal(1);
11185 if (!RHSConstant && DV->isIndirect())
11192 auto *DIExpr = DV->getExpression();
11193 auto NewLocOps = DV->copyLocationOps();
11194 bool Changed =
false;
11195 size_t OrigLocOpsSize = NewLocOps.size();
11196 for (
size_t i = 0; i < OrigLocOpsSize; ++i) {
11201 NewLocOps[i].getSDNode() != &
N)
11212 const auto *TmpDIExpr =
11220 NewLocOps.push_back(
RHS);
11226 assert(Changed &&
"Salvage target doesn't use N");
11229 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
11231 auto AdditionalDependencies = DV->getAdditionalDependencies();
11233 DV->getVariable(), DIExpr, NewLocOps, AdditionalDependencies,
11234 DV->isIndirect(), DV->getDebugLoc(), DV->getOrder(), IsVariadic);
11236 DV->setIsInvalidated();
11237 DV->setIsEmitted();
11239 N0.
getNode()->dumprFull(
this);
11240 dbgs() <<
" into " << *DIExpr <<
'\n');
11247 TypeSize ToSize =
N.getValueSizeInBits(0);
11251 auto NewLocOps = DV->copyLocationOps();
11252 bool Changed =
false;
11253 for (
size_t i = 0; i < NewLocOps.size(); ++i) {
11255 NewLocOps[i].getSDNode() != &
N)
11262 assert(Changed &&
"Salvage target doesn't use N");
11267 DV->getAdditionalDependencies(), DV->isIndirect(),
11268 DV->getDebugLoc(), DV->getOrder(), DV->isVariadic());
11271 DV->setIsInvalidated();
11272 DV->setIsEmitted();
11274 dbgs() <<
" into " << *DbgExpression <<
'\n');
11281 assert(!Dbg->getSDNodes().empty() &&
11282 "Salvaged DbgValue should depend on a new SDNode");
11290 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(
DL) &&
11291 "Expected inlined-at fields to agree");
11307 while (UI != UE &&
N == *UI)
11315 :
SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
11328 "Cannot replace with this method!");
11344 RAUWUpdateListener Listener(*
this, UI, UE);
11349 RemoveNodeFromCSEMaps(
User);
11361 }
while (UI != UE && *UI ==
User);
11364 AddModifiedNodeToCSEMaps(
User);
11380 for (
unsigned i = 0, e =
From->getNumValues(); i != e; ++i)
11383 "Cannot use this version of ReplaceAllUsesWith!");
11391 for (
unsigned i = 0, e =
From->getNumValues(); i != e; ++i)
11392 if (
From->hasAnyUseOfValue(i)) {
11393 assert((i < To->getNumValues()) &&
"Invalid To location");
11402 RAUWUpdateListener Listener(*
this, UI, UE);
11407 RemoveNodeFromCSEMaps(
User);
11419 }
while (UI != UE && *UI ==
User);
11423 AddModifiedNodeToCSEMaps(
User);
11437 if (
From->getNumValues() == 1)
11440 for (
unsigned i = 0, e =
From->getNumValues(); i != e; ++i) {
11450 RAUWUpdateListener Listener(*
this, UI, UE);
11455 RemoveNodeFromCSEMaps(
User);
11461 bool To_IsDivergent =
false;
11468 }
while (UI != UE && *UI ==
User);
11470 if (To_IsDivergent !=
From->isDivergent())
11475 AddModifiedNodeToCSEMaps(
User);
11488 if (
From == To)
return;
11491 if (
From.getNode()->getNumValues() == 1) {
11503 UE =
From.getNode()->use_end();
11504 RAUWUpdateListener Listener(*
this, UI, UE);
11507 bool UserRemovedFromCSEMaps =
false;
11517 if (
Use.getResNo() !=
From.getResNo()) {
11524 if (!UserRemovedFromCSEMaps) {
11525 RemoveNodeFromCSEMaps(
User);
11526 UserRemovedFromCSEMaps =
true;
11533 }
while (UI != UE && *UI ==
User);
11536 if (!UserRemovedFromCSEMaps)
11541 AddModifiedNodeToCSEMaps(
User);
11560bool operator<(
const UseMemo &L,
const UseMemo &R) {
11561 return (intptr_t)L.User < (intptr_t)R.User;
11571 for (UseMemo &Memo :
Uses)
11572 if (Memo.User ==
N)
11573 Memo.User =
nullptr;
11585 switch (
Node->getOpcode()) {
11599 "Conflicting divergence information!");
11604 for (
const auto &
Op :
N->ops()) {
11605 EVT VT =
Op.getValueType();
11608 if (VT != MVT::Other &&
Op.getNode()->isDivergent() &&
11620 if (
N->SDNodeBits.IsDivergent != IsDivergent) {
11621 N->SDNodeBits.IsDivergent = IsDivergent;
11624 }
while (!Worklist.
empty());
11627void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
11629 Order.
reserve(AllNodes.size());
11631 unsigned NOps =
N.getNumOperands();
11634 Order.push_back(&
N);
11636 for (
size_t I = 0;
I != Order.size(); ++
I) {
11638 for (
auto *U :
N->uses()) {
11639 unsigned &UnsortedOps = Degree[U];
11640 if (0 == --UnsortedOps)
11641 Order.push_back(U);
11648 std::vector<SDNode *> TopoOrder;
11649 CreateTopologicalOrder(TopoOrder);
11650 for (
auto *
N : TopoOrder) {
11652 "Divergence bit inconsistency detected");
11675 for (
unsigned i = 0; i != Num; ++i) {
11676 unsigned FromResNo =
From[i].getResNo();
11679 E = FromNode->
use_end(); UI != E; ++UI) {
11681 if (
Use.getResNo() == FromResNo) {
11682 UseMemo Memo = { *UI, i, &
Use };
11683 Uses.push_back(Memo);
11690 RAUOVWUpdateListener Listener(*
this,
Uses);
11692 for (
unsigned UseIndex = 0, UseIndexEnd =
Uses.size();
11693 UseIndex != UseIndexEnd; ) {
11699 if (
User ==
nullptr) {
11705 RemoveNodeFromCSEMaps(
User);
11712 unsigned i =
Uses[UseIndex].Index;
11717 }
while (UseIndex != UseIndexEnd &&
Uses[UseIndex].
User ==
User);
11721 AddModifiedNodeToCSEMaps(
User);
11729 unsigned DAGSize = 0;
11745 unsigned Degree =
N.getNumOperands();
11748 N.setNodeId(DAGSize++);
11750 if (Q != SortedPos)
11751 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
11752 assert(SortedPos != AllNodes.end() &&
"Overran node list");
11756 N.setNodeId(Degree);
11768 unsigned Degree =
P->getNodeId();
11769 assert(Degree != 0 &&
"Invalid node degree");
11773 P->setNodeId(DAGSize++);
11774 if (
P->getIterator() != SortedPos)
11775 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(
P));
11776 assert(SortedPos != AllNodes.end() &&
"Overran node list");
11780 P->setNodeId(Degree);
11783 if (Node.getIterator() == SortedPos) {
11787 dbgs() <<
"Overran sorted position:\n";
11789 dbgs() <<
"Checking if this is due to cycles\n";
11796 assert(SortedPos == AllNodes.end() &&
11797 "Topological sort incomplete!");
11799 "First node in topological sort is not the entry token!");
11800 assert(AllNodes.front().getNodeId() == 0 &&
11801 "First node in topological sort has non-zero id!");
11802 assert(AllNodes.front().getNumOperands() == 0 &&
11803 "First node in topological sort has operands!");
11804 assert(AllNodes.back().getNodeId() == (
int)DAGSize-1 &&
11805 "Last node in topologic sort has unexpected id!");
11806 assert(AllNodes.back().use_empty() &&
11807 "Last node in topologic sort has users!");
11815 for (
SDNode *SD : DB->getSDNodes()) {
11819 SD->setHasDebugValue(
true);
11821 DbgInfo->
add(DB, isParameter);
11828 assert(isa<MemSDNode>(NewMemOpChain) &&
"Expected a memop node");
11834 if (OldChain == NewMemOpChain || OldChain.
use_empty())
11835 return NewMemOpChain;
11838 OldChain, NewMemOpChain);
11841 return TokenFactor;
11846 assert(isa<MemSDNode>(NewMemOp.
getNode()) &&
"Expected a memop node");
11854 assert(isa<ExternalSymbolSDNode>(
Op) &&
"Node should be an ExternalSymbol");
11856 auto *Symbol = cast<ExternalSymbolSDNode>(
Op)->getSymbol();
11860 if (OutFunction !=
nullptr)
11868 std::string ErrorStr;
11870 ErrorFormatter <<
"Undefined external symbol ";
11871 ErrorFormatter <<
'"' << Symbol <<
'"';
11881 return Const !=
nullptr && Const->isZero();
11890 return Const !=
nullptr && Const->isZero() && !Const->isNegative();
11895 return Const !=
nullptr && Const->isAllOnes();
11900 return Const !=
nullptr && Const->isOne();
11905 return Const !=
nullptr && Const->isMinSignedValue();
11909 unsigned OperandNo) {
11914 APInt Const = ConstV->getAPIntValue().trunc(V.getScalarValueSizeInBits());
11920 return Const.isZero();
11922 return Const.isOne();
11925 return Const.isAllOnes();
11927 return Const.isMinSignedValue();
11929 return Const.isMaxSignedValue();
11934 return OperandNo == 1 && Const.isZero();
11937 return OperandNo == 1 && Const.isOne();
11942 return ConstFP->isZero() &&
11943 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
11945 return OperandNo == 1 && ConstFP->isZero() &&
11946 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
11948 return ConstFP->isExactlyValue(1.0);
11950 return OperandNo == 1 && ConstFP->isExactlyValue(1.0);
11954 EVT VT = V.getValueType();
11956 APFloat NeutralAF = !Flags.hasNoNaNs()
11958 : !Flags.hasNoInfs()
11964 return ConstFP->isExactlyValue(NeutralAF);
11973 V = V.getOperand(0);
11978 while (V.getOpcode() ==
ISD::BITCAST && V.getOperand(0).hasOneUse())
11979 V = V.getOperand(0);
11985 V = V.getOperand(0);
11991 V = V.getOperand(0);
11999 unsigned NumBits = V.getScalarValueSizeInBits();
12002 return C && (
C->getAPIntValue().countr_one() >= NumBits);
12006 bool AllowTruncation) {
12007 EVT VT =
N.getValueType();
12016 bool AllowTruncation) {
12023 EVT VecEltVT =
N->getValueType(0).getVectorElementType();
12024 if (
auto *CN = dyn_cast<ConstantSDNode>(
N->getOperand(0))) {
12025 EVT CVT = CN->getValueType(0);
12026 assert(CVT.
bitsGE(VecEltVT) &&
"Illegal splat_vector element extension");
12027 if (AllowTruncation || CVT == VecEltVT)
12034 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
12039 if (CN && (UndefElements.
none() || AllowUndefs)) {
12041 EVT NSVT =
N.getValueType().getScalarType();
12042 assert(CVT.
bitsGE(NSVT) &&
"Illegal build vector element extension");
12043 if (AllowTruncation || (CVT == NSVT))
12052 EVT VT =
N.getValueType();
12060 const APInt &DemandedElts,
12061 bool AllowUndefs) {
12068 BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
12070 if (CN && (UndefElements.
none() || AllowUndefs))
12085 return C &&
C->isZero();
12091 return C &&
C->isOne();
12096 unsigned BitWidth =
N.getScalarValueSizeInBits();
12098 return C &&
C->isAllOnes() &&
C->getValueSizeInBits(0) ==
BitWidth;
12107 :
SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
12131 std::vector<EVT> VTs;
12144const EVT *SDNode::getValueTypeList(
EVT VT) {
12145 static std::set<EVT, EVT::compareRawBits> EVTs;
12146 static EVTArray SimpleVTArray;
12151 return &(*EVTs.insert(VT).first);
12165 if (UI.getUse().getResNo() ==
Value) {
12182 if (UI.getUse().getResNo() ==
Value)
12220 return any_of(
N->op_values(),
12221 [
this](
SDValue Op) { return this == Op.getNode(); });
12235 unsigned Depth)
const {
12236 if (*
this == Dest)
return true;
12240 if (
Depth == 0)
return false;
12260 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
12265 if (
LoadSDNode *Ld = dyn_cast<LoadSDNode>(*
this)) {
12266 if (Ld->isUnordered())
12267 return Ld->getChain().reachesChainWithoutSideEffects(Dest,
Depth-1);
12286 bool AllowPartials) {
12295 return Op.getOpcode() ==
unsigned(BinOp);
12301 unsigned CandidateBinOp =
Op.getOpcode();
12302 if (
Op.getValueType().isFloatingPoint()) {
12304 switch (CandidateBinOp) {
12306 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
12316 auto PartialReduction = [&](
SDValue Op,
unsigned NumSubElts) {
12317 if (!AllowPartials || !
Op)
12319 EVT OpVT =
Op.getValueType();
12342 unsigned Stages =
Log2_32(
Op.getValueType().getVectorNumElements());
12344 for (
unsigned i = 0; i < Stages; ++i) {
12345 unsigned MaskEnd = (1 << i);
12347 if (
Op.getOpcode() != CandidateBinOp)
12348 return PartialReduction(PrevOp, MaskEnd);
12357 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
12364 return PartialReduction(PrevOp, MaskEnd);
12369 return PartialReduction(PrevOp, MaskEnd);
12376 while (
Op.getOpcode() == CandidateBinOp) {
12377 unsigned NumElts =
Op.getValueType().getVectorNumElements();
12385 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
12386 if (NumSrcElts != (2 * NumElts))
12401 EVT VT =
N->getValueType(0);
12410 else if (NE > ResNE)
12413 if (
N->getNumValues() == 2) {
12416 EVT VT1 =
N->getValueType(1);
12420 for (i = 0; i != NE; ++i) {
12421 for (
unsigned j = 0, e =
N->getNumOperands(); j != e; ++j) {
12422 SDValue Operand =
N->getOperand(j);
12441 assert(
N->getNumValues() == 1 &&
12442 "Can't unroll a vector with multiple results!");
12448 for (i= 0; i != NE; ++i) {
12449 for (
unsigned j = 0, e =
N->getNumOperands(); j != e; ++j) {
12450 SDValue Operand =
N->getOperand(j);
12463 switch (
N->getOpcode()) {
12490 for (; i < ResNE; ++i)
12499 unsigned Opcode =
N->getOpcode();
12503 "Expected an overflow opcode");
12505 EVT ResVT =
N->getValueType(0);
12506 EVT OvVT =
N->getValueType(1);
12515 else if (NE > ResNE)
12527 for (
unsigned i = 0; i < NE; ++i) {
12528 SDValue Res =
getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
12551 if (LD->isVolatile() ||
Base->isVolatile())
12554 if (!LD->isSimple())
12556 if (LD->isIndexed() ||
Base->isIndexed())
12558 if (LD->getChain() !=
Base->getChain())
12560 EVT VT = LD->getMemoryVT();
12568 if (BaseLocDecomp.equalBaseIndex(LocDecomp, *
this,
Offset))
12569 return (Dist * (int64_t)Bytes ==
Offset);
12578 int64_t GVOffset = 0;
12590 int FrameIdx = INT_MIN;
12591 int64_t FrameOffset = 0;
12593 FrameIdx = FI->getIndex();
12595 isa<FrameIndexSDNode>(
Ptr.getOperand(0))) {
12597 FrameIdx = cast<FrameIndexSDNode>(
Ptr.getOperand(0))->getIndex();
12598 FrameOffset =
Ptr.getConstantOperandVal(1);
12601 if (FrameIdx != INT_MIN) {
12606 return std::nullopt;
12616 "Split node must be a scalar type");
12621 return std::make_pair(
Lo,
Hi);
12634 return std::make_pair(LoVT, HiVT);
12642 bool *HiIsEmpty)
const {
12652 "Mixing fixed width and scalable vectors when enveloping a type");
12657 *HiIsEmpty =
false;
12665 return std::make_pair(LoVT, HiVT);
12670std::pair<SDValue, SDValue>
12675 "Splitting vector with an invalid mixture of fixed and scalable "
12678 N.getValueType().getVectorMinNumElements() &&
12679 "More vector elements requested than available!");
12689 return std::make_pair(
Lo,
Hi);
12696 EVT VT =
N.getValueType();
12698 "Expecting the mask to be an evenly-sized vector");
12706 return std::make_pair(
Lo,
Hi);
12711 EVT VT =
N.getValueType();
12720 unsigned Start,
unsigned Count,
12722 EVT VT =
Op.getValueType();
12725 if (EltVT ==
EVT())
12728 for (
unsigned i = Start, e = Start + Count; i != e; ++i) {
12741 return Val.MachineCPVal->getType();
12742 return Val.ConstVal->getType();
12746 unsigned &SplatBitSize,
12747 bool &HasAnyUndefs,
12748 unsigned MinSplatBits,
12749 bool IsBigEndian)
const {
12753 if (MinSplatBits > VecWidth)
12758 SplatValue =
APInt(VecWidth, 0);
12759 SplatUndef =
APInt(VecWidth, 0);
12766 assert(NumOps > 0 &&
"isConstantSplat has 0-size build vector");
12769 for (
unsigned j = 0; j < NumOps; ++j) {
12770 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
12772 unsigned BitPos = j * EltWidth;
12775 SplatUndef.
setBits(BitPos, BitPos + EltWidth);
12776 else if (
auto *CN = dyn_cast<ConstantSDNode>(OpVal))
12777 SplatValue.
insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
12778 else if (
auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
12779 SplatValue.
insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
12786 HasAnyUndefs = (SplatUndef != 0);
12789 while (VecWidth > 8) {
12794 unsigned HalfSize = VecWidth / 2;
12801 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
12802 MinSplatBits > HalfSize)
12805 SplatValue = HighValue | LowValue;
12806 SplatUndef = HighUndef & LowUndef;
12808 VecWidth = HalfSize;
12817 SplatBitSize = VecWidth;
12824 if (UndefElements) {
12825 UndefElements->
clear();
12826 UndefElements->
resize(NumOps);
12832 for (
unsigned i = 0; i != NumOps; ++i) {
12833 if (!DemandedElts[i])
12836 if (
Op.isUndef()) {
12838 (*UndefElements)[i] =
true;
12839 }
else if (!Splatted) {
12841 }
else if (Splatted !=
Op) {
12847 unsigned FirstDemandedIdx = DemandedElts.
countr_zero();
12849 "Can only have a splat without a constant for all undefs.");
12866 if (UndefElements) {
12867 UndefElements->
clear();
12868 UndefElements->
resize(NumOps);
12876 for (
unsigned I = 0;
I != NumOps; ++
I)
12878 (*UndefElements)[
I] =
true;
12881 for (
unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
12882 Sequence.append(SeqLen,
SDValue());
12883 for (
unsigned I = 0;
I != NumOps; ++
I) {
12884 if (!DemandedElts[
I])
12886 SDValue &SeqOp = Sequence[
I % SeqLen];
12888 if (
Op.isUndef()) {
12893 if (SeqOp && !SeqOp.
isUndef() && SeqOp !=
Op) {
12899 if (!Sequence.empty())
12903 assert(Sequence.empty() &&
"Failed to empty non-repeating sequence pattern");
12916 return dyn_cast_or_null<ConstantSDNode>(
12922 return dyn_cast_or_null<ConstantSDNode>(
getSplatValue(UndefElements));
12928 return dyn_cast_or_null<ConstantFPSDNode>(
12934 return dyn_cast_or_null<ConstantFPSDNode>(
getSplatValue(UndefElements));
12941 dyn_cast_or_null<ConstantFPSDNode>(
getSplatValue(UndefElements))) {
12944 const APFloat &APF = CN->getValueAPF();
12950 return IntVal.exactLogBase2();
12956 bool IsLittleEndian,
unsigned DstEltSizeInBits,
12964 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12965 "Invalid bitcast scale");
12970 BitVector SrcUndeElements(NumSrcOps,
false);
12972 for (
unsigned I = 0;
I != NumSrcOps; ++
I) {
12974 if (
Op.isUndef()) {
12975 SrcUndeElements.
set(
I);
12978 auto *CInt = dyn_cast<ConstantSDNode>(
Op);
12979 auto *CFP = dyn_cast<ConstantFPSDNode>(
Op);
12980 assert((CInt || CFP) &&
"Unknown constant");
12981 SrcBitElements[
I] = CInt ? CInt->getAPIntValue().trunc(SrcEltSizeInBits)
12982 : CFP->getValueAPF().bitcastToAPInt();
12986 recastRawBits(IsLittleEndian, DstEltSizeInBits, RawBitElements,
12987 SrcBitElements, UndefElements, SrcUndeElements);
12992 unsigned DstEltSizeInBits,
12997 unsigned NumSrcOps = SrcBitElements.
size();
12998 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
12999 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
13000 "Invalid bitcast scale");
13001 assert(NumSrcOps == SrcUndefElements.
size() &&
13002 "Vector size mismatch");
13004 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
13005 DstUndefElements.
clear();
13006 DstUndefElements.
resize(NumDstOps,
false);
13010 if (SrcEltSizeInBits <= DstEltSizeInBits) {
13011 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
13012 for (
unsigned I = 0;
I != NumDstOps; ++
I) {
13013 DstUndefElements.
set(
I);
13014 APInt &DstBits = DstBitElements[
I];
13015 for (
unsigned J = 0; J != Scale; ++J) {
13016 unsigned Idx = (
I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
13017 if (SrcUndefElements[
Idx])
13019 DstUndefElements.
reset(
I);
13020 const APInt &SrcBits = SrcBitElements[
Idx];
13022 "Illegal constant bitwidths");
13023 DstBits.
insertBits(SrcBits, J * SrcEltSizeInBits);
13030 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
13031 for (
unsigned I = 0;
I != NumSrcOps; ++
I) {
13032 if (SrcUndefElements[
I]) {
13033 DstUndefElements.
set(
I * Scale, (
I + 1) * Scale);
13036 const APInt &SrcBits = SrcBitElements[
I];
13037 for (
unsigned J = 0; J != Scale; ++J) {
13038 unsigned Idx = (
I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
13039 APInt &DstBits = DstBitElements[
Idx];
13040 DstBits = SrcBits.
extractBits(DstEltSizeInBits, J * DstEltSizeInBits);
13047 unsigned Opc =
Op.getOpcode();
13054std::optional<std::pair<APInt, APInt>>
13058 return std::nullopt;
13062 return std::nullopt;
13069 return std::nullopt;
13071 for (
unsigned i = 2; i < NumOps; ++i) {
13073 return std::nullopt;
13076 if (Val != (Start + (Stride * i)))
13077 return std::nullopt;
13080 return std::make_pair(Start, Stride);
13096 for (
int Idx = Mask[i]; i != e; ++i)
13097 if (Mask[i] >= 0 && Mask[i] !=
Idx)
13105 if (isa<ConstantSDNode>(
N))
13106 return N.getNode();
13108 return N.getNode();
13116 isa<ConstantSDNode>(
N.getOperand(0)))
13117 return N.getNode();
13124 if (isa<ConstantFPSDNode>(
N))
13125 return N.getNode();
13128 return N.getNode();
13131 isa<ConstantFPSDNode>(
N.getOperand(0)))
13132 return N.getNode();
13138 bool AllowTruncation)
const {
13141 return std::nullopt;
13143 const APInt &CVal = Const->getAPIntValue();
13150 return std::nullopt;
13156 return std::nullopt;
13164 assert(!Node->OperandList &&
"Node already has operands");
13166 "too many operands to fit into SDNode");
13167 SDUse *Ops = OperandRecycler.allocate(
13170 bool IsDivergent =
false;
13171 for (
unsigned I = 0;
I != Vals.
size(); ++
I) {
13172 Ops[
I].setUser(Node);
13173 Ops[
I].setInitial(Vals[
I]);
13177 if (VT != MVT::Other &&
13179 Ops[
I].
getNode()->isDivergent()) {
13180 IsDivergent =
true;
13184 Node->OperandList = Ops;
13187 Node->SDNodeBits.IsDivergent = IsDivergent;
13195 while (Vals.
size() > Limit) {
13196 unsigned SliceIdx = Vals.
size() - Limit;
13270 const SDLoc &DLoc) {
13275 Entry.Ty =
Ptr.getValueType().getTypeForEVT(*
getContext());
13276 Args.push_back(Entry);
13288 assert(
From && To &&
"Invalid SDNode; empty source SDValue?");
13289 auto I = SDEI.find(
From);
13290 if (
I == SDEI.end())
13295 NodeExtraInfo NEI =
I->second;
13304 SDEI[To] = std::move(NEI);
13323 Leafs.emplace_back(
N);
13326 if (!FromReach.
insert(
N).second)
13334 auto DeepCopyTo = [&](
auto &&Self,
const SDNode *
N) {
13337 if (!Visited.
insert(
N).second)
13342 if (!Self(Self,
Op.getNode()))
13362 for (
const SDNode *
N : StartFrom)
13363 VisitFrom(VisitFrom,
N,
MaxDepth - PrevDepth);
13375 errs() <<
"warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
13376 assert(
false &&
"From subgraph too complex - increase max. MaxDepth?");
13378 SDEI[To] = std::move(NEI);
13392 if (!Visited.
insert(
N).second) {
13393 errs() <<
"Detected cycle in SelectionDAG\n";
13394 dbgs() <<
"Offending node:\n";
13395 N->dumprFull(DAG);
dbgs() <<
"\n";
13411 bool check = force;
13412#ifdef EXPENSIVE_CHECKS
13416 assert(
N &&
"Checking nonexistent SDNode");
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
#define __asan_unpoison_memory_region(p, size)
#define LLVM_LIKELY(EXPR)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Looks at all the uses of the given value Returns the Liveness deduced from the uses of this value Adds all uses that cause the result to be MaybeLive to MaybeLiveRetUses If the result is MaybeLiveUses might be modified but its content should be ignored(since it might not be complete). DeadArgumentEliminationPass
Given that RA is a live propagate it s liveness to any other values it uses(according to Uses). void DeadArgumentEliminationPass
Given that RA is a live value
This file defines the DenseSet and SmallDenseSet classes.
This file contains constants used for implementing Dwarf debug support.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file defines a hash set that can be used to remove duplication of nodes in a graph.
Rewrite Partial Register Uses
static const unsigned MaxDepth
static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB)
static bool shouldLowerMemFuncForSize(const MachineFunction &MF)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
mir Rename Register Operands
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
PowerPC Reduce CR logical Operation
const SmallVectorImpl< MachineOperand > & Cond
Contains matchers for matching SelectionDAG nodes and values.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow)
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo)
Lower the call to 'memset' intrinsic function into a series of store operations.
static std::optional< APInt > FoldValueWithUndef(unsigned Opcode, const APInt &C1, bool IsUndef1, const APInt &C2, bool IsUndef2)
static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step, SelectionDAG &DAG)
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC, SDVTList VTList, ArrayRef< SDValue > OpList)
static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, const TargetLowering &TLI, const ConstantDataArraySlice &Slice)
getMemsetStringVal - Similar to getMemsetValue.
static cl::opt< bool > EnableMemCpyDAGOpt("enable-memcpy-dag-opt", cl::Hidden, cl::init(true), cl::desc("Gang up loads and stores generated by inlining of memcpy"))
static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B)
static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList)
AddNodeIDValueTypes - Value type lists are intern'd so we can represent them solely with their pointe...
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef< int > M)
Swaps the values of N1 and N2.
static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice)
Returns true if memcpy source is constant data.
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo)
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, AAResults *AA)
static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)
AddNodeIDOpcode - Add the node opcode to the NodeID data.
static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike)
static bool doNotCSE(SDNode *N)
doNotCSE - Return true if CSE should not be performed for this node.
static cl::opt< int > MaxLdStGlue("ldstmemcpy-glue-max", cl::desc("Number limit for gluing ld/st of memcpy."), cl::Hidden, cl::init(0))
static void AddNodeIDOperands(FoldingSetNodeID &ID, ArrayRef< SDValue > Ops)
AddNodeIDOperands - Various routines for adding operands to the NodeID data.
static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
Try to simplify vector concatenation to an input value, undef, or build vector.
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, SelectionDAG &DAG, SDValue Ptr, int64_t Offset=0)
InferPointerInfo - If the specified ptr/offset is a frame index, infer a MachinePointerInfo record fr...
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N)
If this is an SDNode with special info, add this info to the NodeID data.
static bool gluePropagatesDivergence(const SDNode *Node)
Return true if a glue output should propagate divergence information.
static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G)
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs)
makeVTList - Return an instance of the SDVTList struct initialized with the specified members.
static void VerifySDNode(SDNode *N, const TargetLowering *TLI)
VerifySDNode - Check the given SDNode. Aborts if it is invalid.
static void checkForCyclesHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallPtrSetImpl< const SDNode * > &Checked, const llvm::SelectionDAG *DAG)
static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SmallVector< SDValue, 32 > &OutChains, unsigned From, unsigned To, SmallVector< SDValue, 16 > &OutLoadChains, SmallVector< SDValue, 16 > &OutStoreChains)
static int isSignedOp(ISD::CondCode Opcode)
For an integer comparison, return 1 if the comparison is a signed operation and 2 if the result is an...
static std::optional< APInt > FoldValue(unsigned Opcode, const APInt &C1, const APInt &C2)
static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, unsigned AS)
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR)
Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
static unsigned getSize(unsigned Kind)
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
opStatus divide(const APFloat &RHS, roundingMode RM)
void copySign(const APFloat &RHS)
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
opStatus subtract(const APFloat &RHS, roundingMode RM)
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
opStatus add(const APFloat &RHS, roundingMode RM)
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus multiply(const APFloat &RHS, roundingMode RM)
opStatus fusedMultiplyAdd(const APFloat &Multiplicand, const APFloat &Addend, roundingMode RM)
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
opStatus mod(const APFloat &RHS)
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
APInt umul_ov(const APInt &RHS, bool &Overflow) const
APInt usub_sat(const APInt &RHS) const
APInt udiv(const APInt &RHS) const
Unsigned division operation.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
unsigned popcount() const
Count the number of bits set.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
APInt getHiBits(unsigned numBits) const
Compute an APInt containing numBits highbits from this APInt.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
APInt abs() const
Get the absolute value.
APInt sadd_sat(const APInt &RHS) const
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
void clearAllBits()
Set every bit to 0.
APInt rotr(unsigned rotateAmt) const
Rotate right by rotateAmt.
APInt reverseBits() const
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
bool sle(const APInt &RHS) const
Signed less or equal comparison.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned getNumSignBits() const
Computes the number of leading bits of this APInt that are equal to its sign bit.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
APInt sshl_sat(const APInt &RHS) const
APInt ushl_sat(const APInt &RHS) const
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
APInt rotl(unsigned rotateAmt) const
Rotate left by rotateAmt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
void clearLowBits(unsigned loBits)
Set bottom loBits bits to 0.
unsigned logBase2() const
APInt uadd_sat(const APInt &RHS) const
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
void setAllBits()
Set every bit to 1.
APInt srem(const APInt &RHS) const
Function for signed remainder operation.
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
APInt sext(unsigned width) const
Sign extend to a new width.
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
APInt shl(unsigned shiftAmt) const
Left-shift function.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
bool isOne() const
Determine if this is a value of 1.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
APInt ssub_sat(const APInt &RHS) const
An arbitrary precision integer that knows its signedness.
Recycle small arrays allocated from a BumpPtrAllocator.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
This is an SDNode representing atomic operations.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
void clear()
clear - Removes all bits from the bitvector.
bool none() const
none - Returns true if none of the bits are set.
size_type size() const
size - Returns the number of bits in this bitvector.
int64_t getOffset() const
unsigned getTargetFlags() const
const BlockAddress * getBlockAddress() const
The address of a basic block.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &RawBitElements, BitVector &UndefElements) const
Extract the raw bit data from a build vector of Undef, Constant or ConstantFP node elements.
static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &DstBitElements, ArrayRef< APInt > SrcBitElements, BitVector &DstUndefElements, const BitVector &SrcUndefElements)
Recast bit data SrcBitElements to DstEltSizeInBits wide elements.
bool getRepeatedSequence(const APInt &DemandedElts, SmallVectorImpl< SDValue > &Sequence, BitVector *UndefElements=nullptr) const
Find the shortest repeating sequence of values in the build vector.
ConstantFPSDNode * getConstantFPSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant FP or null if this is not a constant FP splat.
std::optional< std::pair< APInt, APInt > > isConstantSequence() const
If this BuildVector is constant and represents the numerical series "<a, a+n, a+2n,...
SDValue getSplatValue(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted value or a null value if this is not a splat.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
LLVM_ATTRIBUTE_RETURNS_NONNULL void * Allocate(size_t Size, Align Alignment)
Allocate space at the specified alignment.
void Reset()
Deallocate all but the current slab and reset the current pointer to the beginning of it,...
This class represents a function call, abstracting a target machine's calling convention.
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
const APInt & getValue() const
Return the constant as an APInt value reference.
bool isMachineConstantPoolEntry() const
This class represents a range of values.
ConstantRange multiply(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a multiplication of a value in thi...
const APInt * getSingleElement() const
If this set contains a single element, return it, otherwise return null.
static ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
OverflowResult unsignedSubMayOverflow(const ConstantRange &Other) const
Return whether unsigned sub of the two ranges always/never overflows.
OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const
Return whether unsigned add of the two ranges always/never overflows.
KnownBits toKnownBits() const
Return known bits for values in this range.
ConstantRange zeroExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
APInt getSignedMin() const
Return the smallest signed value contained in the ConstantRange.
OverflowResult unsignedMulMayOverflow(const ConstantRange &Other) const
Return whether unsigned mul of the two ranges always/never overflows.
ConstantRange signExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
bool contains(const APInt &Val) const
Return true if the specified value is in the set.
APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
APInt getSignedMax() const
Return the largest signed value contained in the ConstantRange.
OverflowResult
Represents whether an operation on the given constant range is known to always or never overflow.
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
uint32_t getBitWidth() const
Get the bit width of this ConstantRange.
OverflowResult signedSubMayOverflow(const ConstantRange &Other) const
Return whether signed sub of the two ranges always/never overflows.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
Constant * getSplatValue(bool AllowPoison=false) const
If all elements of the vector constant have the same value, return that value.
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
static ExtOps getExtOps(unsigned FromSize, unsigned ToSize, bool Signed)
Returns the ops for a zero- or sign-extension in a DIExpression.
static void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static const DIExpression * convertToVariadicExpression(const DIExpression *Expr)
If Expr is a non-variadic expression (i.e.
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
Base class for variables.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
unsigned getPointerTypeSizeInBits(Type *) const
Layout pointer size, in bits, based on the type.
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Implements a dense probed hash-table based set.
const char * getSymbol() const
unsigned getTargetFlags() const
FoldingSetNodeID - This class is used to gather all the unique data bits of a node.
MachineBasicBlock * MBB
MBB - The current block.
Data structure describing the variable locations in a function.
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
int64_t getOffset() const
unsigned getAddressSpace() const
unsigned getTargetFlags() const
const GlobalValue * getGlobal() const
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
PointerType * getType() const
Global values are always pointers.
This class is used to form a handle around another node that is persistent and is updated across invo...
static bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
constexpr bool isValid() const
This is an important class for using LLVM in a threaded context.
This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate the offet and size that ar...
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
TypeSize getValue() const
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
static MVT getIntegerVT(unsigned BitWidth)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Abstract base class for all machine specific constantpool value subclasses.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
A description of a memory reference used in the backend.
LocationSize getSize() const
Return the size in bytes of the memory reference.
bool isNonTemporal() const
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
bool isDereferenceable() const
This class contains meta information specific to a module.
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt, MachineMemOperand *MMO)
MachineMemOperand * MMO
Memory reference information.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
unsigned getRawSubclassData() const
Return the SubclassData value, without HasDebugValue.
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
A Module instance is used to store all the information related to an LLVM module.
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Pass interface - Implemented by all 'passes'.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
unsigned getAddressSpace() const
Return the address space of the Pointer type.
bool isNull() const
Test if the pointer held in the union is null, regardless of which type it is.
Analysis providing profile information.
void Deallocate(SubClass *E)
Deallocate - Release storage for the pointed-to object.
Keeps track of dbg_value information through SDISel.
BumpPtrAllocator & getAlloc()
void add(SDDbgValue *V, bool isParameter)
void erase(const SDNode *Node)
Invalidate all DbgValues attached to the node and remove it from the Node-to-DbgValues map.
ArrayRef< SDDbgValue * > getSDDbgValues(const SDNode *Node) const
Holds the information from a dbg_label node through SDISel.
Holds the information for a single machine location through SDISel; either an SDNode,...
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(unsigned VReg)
static SDDbgOperand fromConst(const Value *Const)
@ SDNODE
Value is the result of an expression.
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
unsigned getIROrder() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
void dumprFull(const SelectionDAG *G=nullptr) const
printrFull to dbgs().
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
static constexpr size_t getMaxNumOperands()
Return the maximum number of operands that a SDNode can hold.
MemSDNodeBitfields MemSDNodeBits
void Profile(FoldingSetNodeID &ID) const
Gather unique data for the node.
bool getHasDebugValue() const
SDNodeFlags getFlags() const
void setNodeId(int Id)
Set unique node id.
void intersectFlagsWith(const SDNodeFlags Flags)
Clear any flags in this node that aren't also set in Flags.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
static bool areOnlyUsersOf(ArrayRef< const SDNode * > Nodes, const SDNode *N)
Return true if all the users of N are contained in Nodes.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
bool hasPredecessor(const SDNode *N) const
Return true if N is a predecessor of this node.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
void DropOperands()
Release the operands and set this node to have zero operands.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
virtual SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo) const
Emit target-specific code that performs a memset.
virtual SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memmove.
virtual SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memcpy.
SDNodeFlags getFlags() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
SDValue getVPZeroExtendInReg(SDValue Op, SDValue Mask, SDValue EVL, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op)
Return the specified value casted to the target's desired shift amount type.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsExpanding=false)
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
OverflowKind computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const
Determine if the result of the unsigned sub of 2 nodes can overflow.
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
bool isKnownNeverSNaN(SDValue Op, unsigned Depth=0) const
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
void updateDivergence(SDNode *N)
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SelectionDAG(const TargetMachine &TM, CodeGenOptLevel)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getSymbolFunctionGlobalAddress(SDValue Op, Function **TargetFunction=nullptr)
Return a GlobalAddress of the function from the current module with name matching the given ExternalS...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
OverflowKind
Used to represent the possible overflow behavior of an operation.
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
bool cannotBeOrderedNegativeFP(SDValue Op) const
Test whether the given float value is known to be positive.
bool calculateDivergence(SDNode *N)
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
std::optional< uint64_t > getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
void VerifyDAGDivergence()
bool shouldOptForSize() const
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool isEqualTo(SDValue A, SDValue B) const
Test whether two SDValues are known to compare equal.
static constexpr unsigned MaxRecursionDepth
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue expandVACopy(SDNode *Node)
Expand the specified ISD::VACOPY node as the Legalize pass would.
SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
For each demanded element of a vector, see if it is known to be zero.
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
void salvageDebugInfo(SDNode &N)
To be invoked on an SDNode that is slated to be erased.
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
std::pair< SDValue, SDValue > UnrollVectorOverflowOp(SDNode *N, unsigned ResNE=0)
Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDNode * isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
SDValue expandVAArg(SDNode *Node)
Expand the specified ISD::VAARG node as the Legalize pass would.
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
SDValue getMaskedHistogram(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
OverflowKind computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const
Determine if the result of the unsigned mul of 2 nodes can overflow.
void copyExtraInfo(SDNode *From, SDNode *To)
Copy extra info associated with one node to another.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, SDValue InChain, const SDLoc &DLoc)
Helper used to make a call to a library function that has one argument of pointer type.
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly=false, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
OverflowKind computeOverflowForSignedMul(SDValue N0, SDValue N1) const
Determine if the result of the signed mul of 2 nodes can overflow.
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if '(Op & Mask) == Mask'.
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
SDValue getRegister(unsigned Reg, EVT VT)
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
bool isConstantValueOfAnyType(SDValue N) const
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
bool isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
SDValue getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getVPLogicalNOT(const SDLoc &DL, SDValue Val, SDValue Mask, SDValue EVL, EVT VT)
Create a vector-predicated logical NOT operation as (VP_XOR Val, BooleanOne, Mask,...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
iterator_range< allnodes_iterator > allnodes()
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue WidenVector(const SDValue &N, const SDLoc &DL)
Widen the vector up to the next power of two using INSERT_SUBVECTOR.
bool isKnownNeverZeroFloat(SDValue Op) const
Test whether the given floating point SDValue is known to never be positive or negative zero.
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
ArrayRef< SDDbgValue * > GetDbgValues(const SDNode *SD) const
Get the debug values which reference the given SDNode.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
OverflowKind computeOverflowForSignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the signed addition of 2 nodes can overflow.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
ilist< SDNode >::size_type allnodes_size() const
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsCompressing=false)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if 'Op' is known to be zero in DemandedElts.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, bool PoisonOnly=false, bool ConsiderFlags=true, unsigned Depth=0) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
OverflowKind computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the unsigned addition of 2 nodes can overflow.
std::optional< uint64_t > getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT SVT, MachineMemOperand *MMO, bool IsCompressing=false)
void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, SDValue &N2) const
Swap N1 and N2 if Opcode is a commutative binary opcode and the canonical form expects the opposite o...
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
bool isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth=0) const
Test if the given fp value is known to be an integer power-of-2, either positive or negative.
OverflowKind computeOverflowForSignedSub(SDValue N0, SDValue N1) const
Determine if the result of the signed sub of 2 nodes can overflow.
std::optional< uint64_t > getValidShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has a uniform shift amount that is less than the element bit-width of the shi...
LLVMContext * getContext() const
SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
std::pair< EVT, EVT > GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, bool *HiIsEmpty) const
Compute the VTs needed for the low/hi parts of a type, dependent on an enveloping VT that has been sp...
SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
Fold floating-point operations when all operands are constants and/or undefined.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, UniformityInfo *UA, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin, MachineModuleInfo &MMI, FunctionVarLocs const *FnVarLocs)
Prepare this SelectionDAG to process code in the given MachineFunction.
std::optional< ConstantRange > getValidShiftAmountRange(SDValue V, const APInt &DemandedElts, unsigned Depth) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
SDValue FoldSymbolOffset(unsigned Opcode, EVT VT, const GlobalAddressSDNode *GA, const SDNode *N2)
std::optional< bool > isBoolConstant(SDValue N, bool AllowTruncation=false) const
Check if a value \op N is a constant using the target's BooleanContent for its type.
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, ArrayRef< ISD::NodeType > CandidateBinOps, bool AllowPartials=false)
Match a binop + shuffle pyramid that represents a horizontal reduction over the elements of a vector ...
bool isADDLike(SDValue Op, bool NoWrap=false) const
Return true if the specified operand is an ISD::OR or ISD::XOR node that can be treated as an ISD::AD...
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
ilist< SDNode >::iterator allnodes_iterator
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
ArrayRef< int > getMask() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Completely target-dependent object reference.
int64_t getOffset() const
unsigned getTargetFlags() const
Provides information about what library functions are available for the current target.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const
Return true if it is beneficial to convert a load of a constant to just the constant itself.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
const TargetMachine & getTargetMachine() const
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
unsigned getMaxStoresPerMemcpy(bool OptSize) const
Get maximum # of store operations permitted for llvm.memcpy.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
virtual bool shallExtractConstSplatVectorElementToStore(Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const
Return true if the target shall perform extract vector element and store given that the vector is kno...
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
unsigned getMaxStoresPerMemmove(bool OptSize) const
Get maximum # of store operations permitted for llvm.memmove.
virtual unsigned getMaxGluedStoresPerMemcpy() const
Get maximum # of store operations to be glued together.
Align getMinStackArgumentAlignment() const
Return the minimum stack alignment of an argument.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
virtual bool hasVectorBlend() const
Return true if the target has a vector blend instruction.
unsigned getMaxStoresPerMemset(bool OptSize) const
Get maximum # of store operations permitted for llvm.memset.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual bool isLegalStoreImmediate(int64_t Value) const
Return true if the specified immediate is legal for the value input of a store instruction.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual void computeKnownBitsForFrameIndex(int FIOp, KnownBits &Known, const MachineFunction &MF) const
Determine which of the bits of FrameIndex FIOp are known to be 0.
virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
virtual void verifyTargetSDNode(const SDNode *N) const
Check the given SDNode. Aborts if it is invalid.
virtual bool findOptimalMemOpLowering(std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes) const
Determines the optimal series of memory ops to replace the memset / memcpy.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual bool isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const
If SNaN is false,.
virtual void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
virtual bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, UniformityInfo *UA) const
virtual bool isSDNodeAlwaysUniform(const SDNode *N) const
virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth=0) const
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
virtual const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const
This method returns the constant pool value that will be loaded by LD.
virtual bool isGAPlusOffset(SDNode *N, const GlobalValue *&GA, int64_t &Offset) const
Returns true (and the GlobalValue and the offset) if the node is a GlobalAddress + offset.
virtual bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const SelectionDAGTargetInfo * getSelectionDAGInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetLowering * getTargetLowering() const
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
A Use represents the edge between a Value definition and its users.
This class is used to represent an VP_GATHER node.
This class is used to represent a VP_LOAD node.
This class is used to represent an VP_SCATTER node.
This class is used to represent a VP_STORE node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr bool isKnownEven() const
A return value of true indicates we know at compile time that the number of elements (vscale * Min) i...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
A raw_ostream that writes to an std::string.
SmartMutex - A mutex with a compile time constant parameter that indicates whether this mutex should ...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt mulhu(const APInt &C1, const APInt &C2)
Performs (2*N)-bit multiplication on zero-extended operands.
const APInt abdu(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be unsigned.
APInt avgCeilU(const APInt &C1, const APInt &C2)
Compute the ceil of the unsigned average of C1 and C2.
APInt avgFloorU(const APInt &C1, const APInt &C2)
Compute the floor of the unsigned average of C1 and C2.
const APInt abds(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be signed.
APInt mulhs(const APInt &C1, const APInt &C2)
Performs (2*N)-bit multiplication on sign-extended operands.
APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
APInt avgFloorS(const APInt &C1, const APInt &C2)
Compute the floor of the signed average of C1 and C2.
APInt avgCeilS(const APInt &C1, const APInt &C2)
Compute the ceil of the signed average of C1 and C2.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
CondCode getSetCCInverse(CondCode Operation, bool isIntegerLike)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ JUMP_TABLE_DEBUG_INFO
JUMP_TABLE_DEBUG_INFO - Jumptable debug info.
@ BSWAP
Byte Swap and Counting operators.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SRCVALUE
SRCVALUE - This is a node type that holds a Value* that is used to make reference to a value in the L...
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ TargetIndex
TargetIndex - Like a constant pool entry, but with completely target-dependent semantics.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ LIFETIME_START
This corresponds to the llvm.lifetime.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ EXPERIMENTAL_VECTOR_HISTOGRAM
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ SET_FPENV_MEM
Sets the current floating point environment.
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
unsigned getVPForBaseOpcode(unsigned Opcode)
Translate this non-VP Opcode to its corresponding VP Opcode.
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantSDNode predicate.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
bool matchUnaryFpPredicate(SDValue Op, std::function< bool(ConstantFPSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantFPSDNode predicate.
static const int FIRST_TARGET_MEMORY_OPCODE
FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations which do not reference a specific me...
bool isExtOpcode(unsigned Opcode)
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
bool isVectorShrinkable(const SDNode *N, unsigned NewEltSize, bool Signed)
Returns true if the specified node is a vector where all elements can be truncated to the specified e...
bool matchUnaryPredicateImpl(SDValue Op, std::function< bool(ConstNodeType *)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant BUI...
bool isVPBinaryOp(unsigned Opcode)
Whether this is a vector-predicated binary operation opcode.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getBaseOpcodeForVP(unsigned Opcode, bool hasFPExcept)
Translate this VP Opcode to its corresponding non-VP Opcode.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
unsigned getUnorderedFlavor(CondCode Cond)
This function returns 0 if the condition is always false if an operand is a NaN, 1 if the condition i...
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
bool isFreezeUndef(const SDNode *N)
Return true if the specified node is FREEZE(UNDEF).
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
bool isVPReduction(unsigned Opcode)
Whether this is a vector-predicated reduction opcode.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given e...
bool sd_match(SDNode *N, const SelectionDAG *DAG, Pattern &&P)
initializer< Ty > init(const Ty &Val)
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
std::lock_guard< SmartMutex< mt_only > > SmartScopedLock
This is an optimization pass for GlobalISel generic memory operations.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
bool operator<(int64_t V1, const APSInt &V2)
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
SDValue peekThroughExtractSubvectors(SDValue V)
Return the non-extracted vector source operand of V if it exists.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs)
If V is a bitwise not, returns the inverted operand.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
bool getConstantDataArrayInfo(const Value *V, ConstantDataArraySlice &Slice, unsigned ElementSize, uint64_t Offset=0)
Returns true if the value V is a pointer into a ConstantDataArray.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximum semantics.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
MaybeAlign getAlign(const Function &F, unsigned Index)
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
bool isMinSignedConstant(SDValue V)
Returns true if V is a constant min signed integer value.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2019 maximumNumber semantics.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void checkForCycles(const SelectionDAG *DAG, bool force=false)
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
SDValue peekThroughTruncates(SDValue V)
Return the non-truncated source operand of V if it exists.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
CodeGenOptLevel
Code generation optimization level.
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2019 minimumNumber semantics.
@ Mul
Product of integers.
bool isNullConstantOrUndef(SDValue V)
Returns true if V is a constant integer zero or an UNDEF node.
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
OutputIt copy(R &&Range, OutputIt Out)
constexpr unsigned BitWidth
bool funcReturnsFirstArgOfCall(const CallInst &CI)
Returns true if the parent of CI returns CI's first argument after calling CI.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
void computeKnownBitsFromRangeMetadata(const MDNode &Ranges, KnownBits &Known)
Compute known bits from the range metadata.
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimum semantics.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
MDNode * TBAAStruct
The tag for type-based alias analysis (tbaa struct).
MDNode * TBAA
The tag for type-based alias analysis.
static const fltSemantics & IEEEsingle() LLVM_READNONE
cmpResult
IEEE-754R 5.11: Floating Point Comparison Relations.
static constexpr roundingMode rmTowardNegative
static constexpr roundingMode rmNearestTiesToEven
static constexpr roundingMode rmTowardZero
static const fltSemantics & IEEEquad() LLVM_READNONE
static const fltSemantics & IEEEdouble() LLVM_READNONE
static const fltSemantics & IEEEhalf() LLVM_READNONE
static constexpr roundingMode rmTowardPositive
static const fltSemantics & BFloat() LLVM_READNONE
opStatus
IEEE-754R 7: Default exception handling.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Represents offset+length into a ConstantDataArray.
uint64_t Length
Length of the slice.
uint64_t Offset
Slice starts at this Offset.
void move(uint64_t Delta)
Moves the Offset and adjusts Length accordingly.
const ConstantDataArray * Array
ConstantDataArray pointer.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
intptr_t getRawBits() const
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
KnownBits sextInReg(unsigned SrcBitWidth) const
Return known bits for a in-register sign extension of the value we're tracking.
static KnownBits mulhu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from zero-extended multiply-hi.
unsigned countMinSignBits() const
Returns the number of times the sign bit is replicated into the other bits.
static KnownBits smax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smax(LHS, RHS).
bool isNonNegative() const
Returns true if this value is known to be non-negative.
bool isZero() const
Returns true if value is all zero.
void makeNonNegative()
Make this value non-negative.
static KnownBits usub_sat(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from llvm.usub.sat(LHS, RHS)
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
static KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
static std::optional< bool > ne(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_NE result.
void makeNegative()
Make this value negative.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
KnownBits byteSwap() const
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
void setAllZero()
Make all bits known to be zero and discard any previous information.
KnownBits reverseBits() const
KnownBits concat(const KnownBits &Lo) const
Concatenate the bits from Lo onto the bottom of *this.
unsigned getBitWidth() const
Get the bit width of this value.
static KnownBits umax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umax(LHS, RHS).
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
KnownBits unionWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for either this or RHS or both.
static KnownBits lshr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for lshr(LHS, RHS).
bool isNonZero() const
Returns true if this value is known to be non-zero.
static KnownBits abdu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for abdu(LHS, RHS).
KnownBits extractBits(unsigned NumBits, unsigned BitPosition) const
Return a subset of the known bits from [bitPosition,bitPosition+numBits).
static KnownBits avgFloorU(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgFloorU.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits computeForSubBorrow(const KnownBits &LHS, KnownBits RHS, const KnownBits &Borrow)
Compute known bits results from subtracting RHS from LHS with 1-bit Borrow.
KnownBits zextOrTrunc(unsigned BitWidth) const
Return known bits for a zero extension or truncation of the value we're tracking.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
static KnownBits abds(KnownBits LHS, KnownBits RHS)
Compute known bits for abds(LHS, RHS).
static KnownBits smin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smin(LHS, RHS).
static KnownBits mulhs(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from sign-extended multiply-hi.
static KnownBits srem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for srem(LHS, RHS).
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
bool isStrictlyPositive() const
Returns true if this value is known to be positive.
static KnownBits sdiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for sdiv(LHS, RHS).
static KnownBits avgFloorS(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgFloorS.
static bool haveNoCommonBitsSet(const KnownBits &LHS, const KnownBits &RHS)
Return true if LHS and RHS have no common bits set.
bool isNegative() const
Returns true if this value is known to be negative.
static KnownBits computeForAddCarry(const KnownBits &LHS, const KnownBits &RHS, const KnownBits &Carry)
Compute known bits resulting from adding LHS, RHS and a 1-bit Carry.
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
void insertBits(const KnownBits &SubBits, unsigned BitPosition)
Insert the bits from a smaller known bits starting at bitPosition.
static KnownBits avgCeilU(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgCeilU.
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
KnownBits anyext(unsigned BitWidth) const
Return known bits for an "any" extension of the value we're tracking, where we don't know anything ab...
KnownBits abs(bool IntMinIsPoison=false) const
Compute known bits for the absolute value.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
static KnownBits umin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umin(LHS, RHS).
static KnownBits avgCeilS(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgCeilS.
This class contains a discriminated union of information about pointers in memory operands,...
bool isDereferenceable(unsigned Size, LLVMContext &C, const DataLayout &DL) const
Return true if memory region [V, V+Offset+Size) is known to be dereferenceable.
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
PointerUnion< const Value *, const PseudoSourceValue * > V
This is the IR pointer value for the access, or it is null if unknown.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, bool IsZeroMemset, bool IsVolatile)
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc=false)
These are IR-level optimization flags that may be propagated to SDNodes.
void intersectWith(const SDNodeFlags Flags)
Clear any flags in this flag set that aren't also set in Flags.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
DAGUpdateListener *const Next
virtual void NodeDeleted(SDNode *N, SDNode *E)
The node N that was deleted and, if E is not null, an equivalent node E that replaced it.
virtual void NodeInserted(SDNode *N)
The node N that was inserted.
virtual void NodeUpdated(SDNode *N)
The node N that was updated.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)