102void SelectionDAG::DAGNodeDeletedListener::anchor() {}
103void SelectionDAG::DAGNodeInsertedListener::anchor() {}
105#define DEBUG_TYPE "selectiondag"
145 if (
auto OptAPInt = V.getOperand(0)->bitcastToAPInt()) {
147 V.getValueType().getVectorElementType().getSizeInBits();
148 SplatVal = OptAPInt->
trunc(EltSize);
158 unsigned SplatBitSize;
160 unsigned EltSize = V.getValueType().getVectorElementType().getSizeInBits();
165 const bool IsBigEndian =
false;
166 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
167 EltSize, IsBigEndian) &&
168 EltSize == SplatBitSize;
187 unsigned i = 0, e = V.getNumOperands();
190 while (i != e && V.getOperand(i).isUndef())
194 if (i == e)
return false;
206 unsigned EltSize = V.getValueType().getScalarSizeInBits();
207 if (OptAPInt->countr_one() < EltSize)
215 for (++i; i != e; ++i)
234 bool IsAllUndef =
true;
235 for (
const SDValue &
Op : V->op_values()) {
247 if (
auto OptAPInt =
Op->bitcastToAPInt()) {
248 unsigned EltSize = V.getValueType().getScalarSizeInBits();
249 if (OptAPInt->countr_zero() < EltSize)
273 for (
const SDValue &
Op : V->op_values()) {
286 for (
const SDValue &
Op : V->op_values()) {
296 assert(V.getValueType().isVector() &&
"Expected a vector!");
298 unsigned EltSize = V.getValueType().getScalarSizeInBits();
299 if (EltSize <= NewEltSize)
303 return (V.getOperand(0).getValueType().getScalarSizeInBits() <=
308 return (V.getOperand(0).getValueType().getScalarSizeInBits() <=
315 for (
const SDValue &
Op : V->op_values()) {
321 APInt C =
Op->getAsAPIntVal().trunc(EltSize);
322 if (
Signed &&
C.trunc(NewEltSize).sext(EltSize) !=
C)
324 if (!
Signed &&
C.trunc(NewEltSize).zext(EltSize) !=
C)
335 if (V.getNumOperands() == 0)
337 return all_of(V->op_values(), [](
SDValue Op) { return Op.isUndef(); });
341 return V.getOpcode() ==
ISD::FREEZE && V.getOperand(0).isUndef();
344template <
typename ConstNodeType>
346 std::function<
bool(ConstNodeType *)> Match,
347 bool AllowUndefs,
bool AllowTruncation) {
360 EVT SVT =
Op.getValueType().getScalarType();
361 for (
unsigned i = 0, e =
Op.getNumOperands(); i != e; ++i) {
365 if (AllowUndefs &&
Op.getOperand(i).isUndef()) {
372 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
388 bool AllowUndefs,
bool AllowTypeMismatch) {
389 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
395 return Match(LHSCst, RHSCst);
398 if (LHS.getOpcode() != RHS.getOpcode() ||
407 for (
unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
412 bool LHSUndef = AllowUndefs && LHSOp.
isUndef();
413 bool RHSUndef = AllowUndefs && RHSOp.
isUndef();
416 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
418 if (!AllowTypeMismatch && (LHSOp.
getValueType() != SVT ||
421 if (!Match(LHSCst, RHSCst))
458 switch (VecReduceOpcode) {
463 case ISD::VP_REDUCE_FADD:
464 case ISD::VP_REDUCE_SEQ_FADD:
468 case ISD::VP_REDUCE_FMUL:
469 case ISD::VP_REDUCE_SEQ_FMUL:
472 case ISD::VP_REDUCE_ADD:
475 case ISD::VP_REDUCE_MUL:
478 case ISD::VP_REDUCE_AND:
481 case ISD::VP_REDUCE_OR:
484 case ISD::VP_REDUCE_XOR:
487 case ISD::VP_REDUCE_SMAX:
490 case ISD::VP_REDUCE_SMIN:
493 case ISD::VP_REDUCE_UMAX:
496 case ISD::VP_REDUCE_UMIN:
499 case ISD::VP_REDUCE_FMAX:
502 case ISD::VP_REDUCE_FMIN:
505 case ISD::VP_REDUCE_FMAXIMUM:
508 case ISD::VP_REDUCE_FMINIMUM:
536#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
539#include "llvm/IR/VPIntrinsics.def"
547#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
548#define VP_PROPERTY_BINARYOP return true;
549#define END_REGISTER_VP_SDNODE(VPSD) break;
550#include "llvm/IR/VPIntrinsics.def"
559 case ISD::VP_REDUCE_ADD:
560 case ISD::VP_REDUCE_MUL:
561 case ISD::VP_REDUCE_AND:
562 case ISD::VP_REDUCE_OR:
563 case ISD::VP_REDUCE_XOR:
564 case ISD::VP_REDUCE_SMAX:
565 case ISD::VP_REDUCE_SMIN:
566 case ISD::VP_REDUCE_UMAX:
567 case ISD::VP_REDUCE_UMIN:
568 case ISD::VP_REDUCE_FMAX:
569 case ISD::VP_REDUCE_FMIN:
570 case ISD::VP_REDUCE_FMAXIMUM:
571 case ISD::VP_REDUCE_FMINIMUM:
572 case ISD::VP_REDUCE_FADD:
573 case ISD::VP_REDUCE_FMUL:
574 case ISD::VP_REDUCE_SEQ_FADD:
575 case ISD::VP_REDUCE_SEQ_FMUL:
585#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
588#include "llvm/IR/VPIntrinsics.def"
597#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
600#include "llvm/IR/VPIntrinsics.def"
610#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
611#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
612#define END_REGISTER_VP_SDNODE(VPOPC) break;
613#include "llvm/IR/VPIntrinsics.def"
622#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
623#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
624#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
625#include "llvm/IR/VPIntrinsics.def"
647 unsigned OldL = (Operation >> 2) & 1;
648 unsigned OldG = (Operation >> 1) & 1;
672 bool isIntegerLike) {
697 bool IsInteger =
Type.isInteger();
702 unsigned Op = Op1 | Op2;
718 bool IsInteger =
Type.isInteger();
764 ID.AddPointer(
C->getConstantIntValue());
765 ID.AddBoolean(
C->isOpaque());
836 ID.AddInteger(LD->getMemoryVT().getRawBits());
837 ID.AddInteger(LD->getRawSubclassData());
838 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
839 ID.AddInteger(LD->getMemOperand()->getFlags());
844 ID.AddInteger(ST->getMemoryVT().getRawBits());
845 ID.AddInteger(ST->getRawSubclassData());
846 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
847 ID.AddInteger(ST->getMemOperand()->getFlags());
858 case ISD::VP_LOAD_FF: {
860 ID.AddInteger(LD->getMemoryVT().getRawBits());
861 ID.AddInteger(LD->getRawSubclassData());
862 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
863 ID.AddInteger(LD->getMemOperand()->getFlags());
866 case ISD::VP_STORE: {
874 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
881 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
888 case ISD::VP_GATHER: {
896 case ISD::VP_SCATTER: {
933 ID.AddInteger(M->getMemoryVT().getRawBits());
934 ID.AddInteger(M->getRawSubclassData());
935 ID.AddInteger(M->getPointerInfo().getAddrSpace());
936 ID.AddInteger(M->getMemOperand()->getFlags());
973 ID.AddInteger(AT->getMemoryVT().getRawBits());
974 ID.AddInteger(AT->getRawSubclassData());
975 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
976 ID.AddInteger(AT->getMemOperand()->getFlags());
982 ID.AddInteger(MN->getRawSubclassData());
983 ID.AddInteger(MN->getMemoryVT().getRawBits());
985 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
986 ID.AddInteger(MMO->getFlags());
1004 ID.AddInteger(
N->getOpcode());
1005 ID.AddPointer(
N->getVTList().VTs);
1007 ID.AddPointer(
Op.getNode());
1008 ID.AddInteger(
Op.getResNo());
1014 ID.AddInteger(
Key.Opcode);
1015 ID.AddPointer(
Key.VTs);
1017 ID.AddPointer(
Op.getNode());
1018 ID.AddInteger(
Op.getResNo());
1020 ID.AddNodeID(
Key.Tail);
1025 if (
N.getOpcode() !=
Key.Opcode ||
N.getVTList().VTs !=
Key.VTs)
1033 return Tail ==
Key.Tail;
1043 assert((NodeProfile == KeyProfile) == Result &&
1044 "SDNodeKey equality disagrees with profile");
1055 if (
N->getValueType(0) == MVT::Glue)
1058 switch (
N->getOpcode()) {
1066 for (
unsigned i = 1, e =
N->getNumValues(); i != e; ++i)
1067 if (
N->getValueType(i) == MVT::Glue)
1076 EVT VT =
V.getValueType();
1095 if (
Node.use_empty())
1110 while (!DeadNodes.
empty()) {
1119 DUL->NodeDeleted(
N,
nullptr);
1122 RemoveNodeFromCSEMaps(
N);
1153 RemoveNodeFromCSEMaps(
N);
1157 DeleteNodeNotInCSEMaps(
N);
1160void SelectionDAG::DeleteNodeNotInCSEMaps(
SDNode *
N) {
1161 assert(
N->getIterator() != AllNodes.begin() &&
1162 "Cannot delete the entry node!");
1163 assert(
N->use_empty() &&
"Cannot delete a node that is not dead!");
1172 assert(!(V->isVariadic() && isParameter));
1174 ByvalParmDbgValues.push_back(V);
1176 DbgValues.push_back(V);
1179 DbgValMap[
Node].push_back(V);
1183 DbgValMapType::iterator
I = DbgValMap.find(
Node);
1184 if (
I == DbgValMap.end())
1186 for (
auto &Val:
I->second)
1187 Val->setIsInvalidated();
1191void SelectionDAG::DeallocateNode(
SDNode *
N) {
1214void SelectionDAG::verifyNode(
SDNode *
N)
const {
1215 switch (
N->getOpcode()) {
1217 if (
N->isTargetOpcode())
1221 EVT VT =
N->getValueType(0);
1222 assert(
N->getNumValues() == 1 &&
"Too many results!");
1224 "Wrong return type!");
1225 assert(
N->getNumOperands() == 2 &&
"Wrong number of operands!");
1226 assert(
N->getOperand(0).getValueType() ==
N->getOperand(1).getValueType() &&
1227 "Mismatched operand types!");
1229 "Wrong operand type!");
1231 "Wrong return type size");
1235 assert(
N->getNumValues() == 1 &&
"Too many results!");
1236 assert(
N->getValueType(0).isVector() &&
"Wrong return type!");
1237 assert(
N->getNumOperands() ==
N->getValueType(0).getVectorNumElements() &&
1238 "Wrong number of operands!");
1239 EVT EltVT =
N->getValueType(0).getVectorElementType();
1240 for (
const SDUse &
Op :
N->ops()) {
1241 assert((
Op.getValueType() == EltVT ||
1242 (EltVT.
isInteger() &&
Op.getValueType().isInteger() &&
1243 EltVT.
bitsLE(
Op.getValueType()))) &&
1244 "Wrong operand type!");
1245 assert(
Op.getValueType() ==
N->getOperand(0).getValueType() &&
1246 "Operands must all have the same type");
1254 assert(
N->getNumValues() == 2 &&
"Wrong number of results!");
1255 assert(
N->getVTList().NumVTs == 2 &&
N->getNumOperands() == 2 &&
1256 "Invalid add/sub overflow op!");
1257 assert(
N->getVTList().VTs[0].isInteger() &&
1258 N->getVTList().VTs[1].isInteger() &&
1259 N->getOperand(0).getValueType() ==
N->getOperand(1).getValueType() &&
1260 N->getOperand(0).getValueType() ==
N->getVTList().VTs[0] &&
1261 "Binary operator types must match!");
1271void SelectionDAG::InsertNode(
SDNode *
N) {
1272 AllNodes.push_back(
N);
1274 N->PersistentId = NextPersistentId++;
1278 DUL->NodeInserted(
N);
1285bool SelectionDAG::RemoveNodeFromCSEMaps(
SDNode *
N) {
1286 bool Erased =
false;
1287 switch (
N->getOpcode()) {
1291 "Cond code doesn't exist!");
1300 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
1306 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1312 Erased = ExtendedValueTypeNodes.erase(VT);
1323 Erased = CSEMap.erase(
N);
1330 if (!Erased &&
N->getValueType(
N->getNumValues()-1) != MVT::Glue &&
1345SelectionDAG::AddModifiedNodeToCSEMaps(
SDNode *
N) {
1349 SDNode *Existing = CSEMap.getOrInsert(
N);
1350 if (Existing !=
N) {
1361 MemNode->refineMMOMetadata(NewMMOs);
1367 DUL->NodeDeleted(
N, Existing);
1368 DeleteNodeNotInCSEMaps(
N);
1375 DUL->NodeUpdated(
N);
1388 SDNodeKey
ID(
N->getOpcode(),
N->getVTList(),
Ops);
1390 SDNode *
Node = lookupNode(ID, SDLoc(
N), InsertToken);
1392 Node->intersectFlagsWith(
N->getFlags());
1406 SDNodeKey
ID(
N->getOpcode(),
N->getVTList(),
Ops);
1408 SDNode *
Node = lookupNode(ID, SDLoc(
N), InsertToken);
1410 Node->intersectFlagsWith(
N->getFlags());
1423 SDNodeKey
ID(
N->getOpcode(),
N->getVTList(),
Ops);
1425 SDNode *
Node = lookupNode(ID, SDLoc(
N), InsertToken);
1427 Node->intersectFlagsWith(
N->getFlags());
1440 : TM(tm), OptLevel(OL), EntryNode(
ISD::EntryToken, 0,
DebugLoc(),
1443 InsertNode(&EntryNode);
1456 LibInfo = LibraryInfo;
1457 Libcalls = LibcallsInfo;
1458 Context = &MF->getFunction().getContext();
1459 DAGOpts = &Context->getLibraryOptions().get<SelectionDAGOptions>();
1463 FnVarLocs = VarLocs;
1467 assert(!UpdateListeners &&
"Dangling registered DAGUpdateListeners");
1469 OperandRecycler.clear(OperandAllocator);
1478 return FLI->MBB->getBasicBlock();
1481void SelectionDAG::allnodes_clear() {
1482 assert(&*AllNodes.begin() == &EntryNode);
1483 AllNodes.remove(AllNodes.begin());
1484 while (!AllNodes.empty())
1485 DeallocateNode(&AllNodes.front());
1487 NextPersistentId = 0;
1495 switch (
N->getOpcode()) {
1500 "debug location. Use another overload.");
1508 SDNode *
N = CSEMap.lookup(
Key, InsertToken);
1510 switch (
N->getOpcode()) {
1516 if (
N->getDebugLoc() !=
DL.getDebugLoc())
1523 if (
DL.getIROrder() &&
DL.getIROrder() <
N->getIROrder())
1524 N->setDebugLoc(
DL.getDebugLoc());
1533 OperandRecycler.clear(OperandAllocator);
1534 OperandAllocator.Reset();
1537 ExtendedValueTypeNodes.clear();
1538 ExternalSymbols.clear();
1539 TargetExternalSymbols.clear();
1545 EntryNode.UseList =
nullptr;
1546 InsertNode(&EntryNode);
1552 return VT.
bitsGT(
Op.getValueType())
1558std::pair<SDValue, SDValue>
1562 "Strict no-op FP extend/round not allowed.");
1569 return std::pair<SDValue, SDValue>(Res,
SDValue(Res.
getNode(), 1));
1573 return VT.
bitsGT(
Op.getValueType()) ?
1579 return VT.
bitsGT(
Op.getValueType()) ?
1585 return VT.
bitsGT(
Op.getValueType()) ?
1593 auto Type =
Op.getValueType();
1597 auto Size =
Op.getValueSizeInBits();
1611 return getNode(TLI->getExtendForContent(BType), SL, VT,
Op);
1615 EVT OpVT =
Op.getValueType();
1617 "Cannot getZeroExtendInReg FP types");
1619 "getZeroExtendInReg type should be vector iff the operand "
1623 "Vector element counts must match in getZeroExtendInReg");
1669 switch (TLI->getBooleanContents(OpVT)) {
1680 bool isT,
bool isO) {
1686 bool isT,
bool isO) {
1687 return getConstant(*ConstantInt::get(*Context, Val),
DL, VT, isT, isO);
1691 EVT VT,
bool isT,
bool isO) {
1708 EltVT = TLI->getTypeToTransformTo(*
getContext(), EltVT);
1714 Elt = ConstantInt::get(*
getContext(), NewVal);
1726 EVT ViaEltVT = TLI->getTypeToTransformTo(*
getContext(), EltVT);
1733 "Can only handle an even split!");
1737 for (
unsigned i = 0; i != Parts; ++i)
1739 NewVal.
extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits),
DL,
1740 ViaEltVT, isT, isO));
1745 unsigned ViaVecNumElts = VT.
getSizeInBits() / ViaEltSizeInBits;
1756 NewVal.
extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits),
DL,
1757 ViaEltVT, isT, isO));
1762 std::reverse(EltParts.
begin(), EltParts.
end());
1781 "APInt size does not match type size!");
1789 if ((
N = lookupNode(ID,
DL, InsertToken)))
1794 N = newSDNode<ConstantSDNode>(isT, isO, Elt, VTs);
1796 N->setDebugLoc(
DL.getDebugLoc());
1797 CSEMap.insert(
N, InsertToken);
1809 bool isT,
bool isO) {
1817 IsTarget, IsOpaque);
1849 EVT VT,
bool isTarget) {
1869 if ((
N = lookupNode(ID,
DL, InsertToken)))
1874 N = newSDNode<ConstantFPSDNode>(isTarget, Elt, VTs);
1875 CSEMap.insert(
N, InsertToken);
1889 if (EltVT == MVT::f32)
1891 if (EltVT == MVT::f64)
1893 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1894 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1905 EVT VT, int64_t
Offset,
bool isTargetGA,
1906 unsigned TargetFlags) {
1907 assert((TargetFlags == 0 || isTargetGA) &&
1908 "Cannot set target flags on target-independent globals");
1925 ID.AddInteger(TargetFlags);
1927 if (
SDNode *E = lookupNode(ID,
DL, InsertToken))
1930 auto *
N = newSDNode<GlobalAddressSDNode>(
1931 Opc,
DL.getIROrder(),
DL.getDebugLoc(), GV, VTs,
Offset, TargetFlags);
1932 CSEMap.insert(
N, InsertToken);
1942 if (
SDNode *E = lookupNode(ID,
SDLoc(), InsertToken))
1945 auto *
N = newSDNode<DeactivationSymbolSDNode>(GV, VTs);
1946 CSEMap.insert(
N, InsertToken);
1957 if (
SDNode *E = lookupNode(ID, InsertToken))
1960 auto *
N = newSDNode<FrameIndexSDNode>(FI, VTs, isTarget);
1961 CSEMap.insert(
N, InsertToken);
1967 unsigned TargetFlags) {
1968 assert((TargetFlags == 0 || isTarget) &&
1969 "Cannot set target flags on target-independent jump tables");
1974 ID.AddInteger(TargetFlags);
1976 if (
SDNode *E = lookupNode(ID, InsertToken))
1979 auto *
N = newSDNode<JumpTableSDNode>(JTI, VTs, isTarget, TargetFlags);
1980 CSEMap.insert(
N, InsertToken);
1994 bool isTarget,
unsigned TargetFlags) {
1995 assert((TargetFlags == 0 || isTarget) &&
1996 "Cannot set target flags on target-independent globals");
2007 ID.AddInteger(TargetFlags);
2009 if (
SDNode *E = lookupNode(ID, InsertToken))
2012 auto *
N = newSDNode<ConstantPoolSDNode>(isTarget,
C, VTs,
Offset, *Alignment,
2014 CSEMap.insert(
N, InsertToken);
2023 bool isTarget,
unsigned TargetFlags) {
2024 assert((TargetFlags == 0 || isTarget) &&
2025 "Cannot set target flags on target-independent globals");
2033 C->addSelectionDAGCSEId(ID.Tail);
2034 ID.AddInteger(TargetFlags);
2036 if (
SDNode *E = lookupNode(ID, InsertToken))
2039 auto *
N = newSDNode<ConstantPoolSDNode>(isTarget,
C, VTs,
Offset, *Alignment,
2041 CSEMap.insert(
N, InsertToken);
2050 if (
SDNode *E = lookupNode(ID, InsertToken))
2053 auto *
N = newSDNode<BasicBlockSDNode>(
MBB);
2054 CSEMap.insert(
N, InsertToken);
2061 ValueTypeNodes.size())
2068 N = newSDNode<VTSDNode>(VT);
2074 SDNode *&
N = ExternalSymbols[Sym];
2076 N = newSDNode<ExternalSymbolSDNode>(
false, Sym, 0,
getVTList(VT));
2090 N = newSDNode<MCSymbolSDNode>(Sym,
getVTList(VT));
2096 unsigned TargetFlags) {
2098 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2100 N = newSDNode<ExternalSymbolSDNode>(
true, Sym, TargetFlags,
getVTList(VT));
2106 EVT VT,
unsigned TargetFlags) {
2112 if ((
unsigned)
Cond >= CondCodeNodes.size())
2113 CondCodeNodes.resize(
Cond+1);
2115 if (!CondCodeNodes[
Cond]) {
2116 auto *
N = newSDNode<CondCodeSDNode>(
Cond);
2117 CondCodeNodes[
Cond] =
N;
2126 "APInt size does not match type size!");
2144template <
typename Ty>
2146 EVT VT, Ty Quantity) {
2147 if (Quantity.isScalable())
2151 return DAG.
getConstant(Quantity.getKnownMinValue(),
DL, VT);
2177 const APInt &StepVal) {
2201 "Must have the same number of vector elements as mask elements!");
2203 "Invalid VECTOR_SHUFFLE");
2214 int NElts = Mask.size();
2216 [&](
int M) {
return M < (NElts * 2) && M >= -1; }) &&
2217 "Index out of range");
2225 for (
int i = 0; i != NElts; ++i)
2226 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2233 if (TLI->hasVectorBlend()) {
2242 for (
int i = 0; i < NElts; ++i) {
2243 if (MaskVec[i] <
Offset || MaskVec[i] >= (
Offset + NElts))
2247 if (UndefElements[MaskVec[i] -
Offset]) {
2253 if (!UndefElements[i])
2258 BlendSplat(N1BV, 0);
2260 BlendSplat(N2BV, NElts);
2265 bool AllLHS =
true, AllRHS =
true;
2267 for (
int i = 0; i != NElts; ++i) {
2268 if (MaskVec[i] >= NElts) {
2273 }
else if (MaskVec[i] >= 0) {
2277 if (AllLHS && AllRHS)
2279 if (AllLHS && !N2Undef)
2288 if (N1.
isUndef() && N2Undef) {
2295 bool Identity =
true, AllSame =
true;
2296 for (
int i = 0; i != NElts; ++i) {
2297 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity =
false;
2298 if (MaskVec[i] != MaskVec[0]) AllSame =
false;
2300 if (Identity && NElts)
2333 if (AllSame && SameNumElts) {
2334 EVT BuildVT = BV->getValueType(0);
2350 for (
int i = 0; i != NElts; ++i)
2351 ID.AddInteger(MaskVec[i]);
2354 if (
SDNode *E = lookupNode(ID, dl, InsertToken))
2360 int *MaskAlloc = OperandAllocator.Allocate<
int>(NElts);
2363 auto *
N = newSDNode<ShuffleVectorSDNode>(VTs, dl.
getIROrder(),
2365 createOperands(
N,
Ops);
2367 CSEMap.insert(
N, InsertToken);
2375 EVT VT = SV.getValueType(0);
2379 SDValue Op0 = SV.getOperand(0);
2380 SDValue Op1 = SV.getOperand(1);
2389 if (
SDNode *E = lookupNode(ID, InsertToken))
2392 auto *
N = newSDNode<RegisterSDNode>(Reg, VTs);
2393 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(
N, FLI, UA);
2394 CSEMap.insert(
N, InsertToken);
2403 if (
SDNode *E = lookupNode(ID, InsertToken))
2406 auto *
N = newSDNode<RegisterMaskSDNode>(RegMask);
2407 CSEMap.insert(
N, InsertToken);
2421 ID.AddPointer(Label);
2423 if (
SDNode *E = lookupNode(ID, InsertToken))
2428 createOperands(
N,
Ops);
2430 CSEMap.insert(
N, InsertToken);
2436 int64_t
Offset,
bool isTarget,
2437 unsigned TargetFlags) {
2444 ID.AddInteger(TargetFlags);
2446 if (
SDNode *E = lookupNode(ID, InsertToken))
2449 auto *
N = newSDNode<BlockAddressSDNode>(
Opc, VTs, BA,
Offset, TargetFlags);
2450 CSEMap.insert(
N, InsertToken);
2460 if (
SDNode *E = lookupNode(ID, InsertToken))
2463 auto *
N = newSDNode<SrcValueSDNode>(V);
2464 CSEMap.insert(
N, InsertToken);
2474 if (
SDNode *E = lookupNode(ID, InsertToken))
2477 auto *
N = newSDNode<MDNodeSDNode>(MD);
2478 CSEMap.insert(
N, InsertToken);
2484 if (VT == V.getValueType())
2491 unsigned SrcAS,
unsigned DestAS,
2496 ID.AddInteger(SrcAS);
2497 ID.AddInteger(DestAS);
2500 if (
SDNode *E = lookupNode(ID, dl, InsertToken)) {
2501 E->intersectFlagsWith(Flags);
2506 VTs, SrcAS, DestAS);
2508 createOperands(
N,
Ops);
2510 CSEMap.insert(
N, InsertToken);
2531 if (
OpTy == ShTy ||
OpTy.isVector())
return Op;
2540 EVT VT =
Node->getValueType(0);
2549 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2587 Align RedAlign = UseABI ?
DL.getABITypeAlign(Ty) :
DL.getPrefTypeAlign(Ty);
2589 if (TLI->isTypeLegal(VT) || !VT.
isVector())
2597 if (RedAlign > StackAlign) {
2600 unsigned NumIntermediates;
2601 TLI->getVectorTypeBreakdown(*
getContext(), VT, IntermediateVT,
2602 NumIntermediates, RegisterVT);
2604 Align RedAlign2 = UseABI ?
DL.getABITypeAlign(Ty) :
DL.getPrefTypeAlign(Ty);
2605 if (RedAlign2 < RedAlign)
2606 RedAlign = RedAlign2;
2611 RedAlign = std::min(RedAlign, StackAlign);
2626 false,
nullptr, StackID);
2641 "Don't know how to choose the maximum size when creating a stack "
2650 Align Align = std::max(
DL.getPrefTypeAlign(Ty1),
DL.getPrefTypeAlign(Ty2));
2674 if (SrcVT.
bitsGT(SlotVT))
2682 if (SlotVT.
bitsEq(DestVT))
2695 auto GetUndefBooleanConstant = [&]() {
2697 TLI->getBooleanContents(OpVT) ==
2734 return GetUndefBooleanConstant();
2739 return GetUndefBooleanConstant();
2748 const APInt &C2 = N2C->getAPIntValue();
2750 const APInt &
C1 = N1C->getAPIntValue();
2760 if (N1CFP && N2CFP) {
2765 return GetUndefBooleanConstant();
2770 return GetUndefBooleanConstant();
2776 return GetUndefBooleanConstant();
2781 return GetUndefBooleanConstant();
2786 return GetUndefBooleanConstant();
2792 return GetUndefBooleanConstant();
2819 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.
getSimpleVT()))
2821 return getSetCC(dl, VT, N2, N1, SwappedCond, {},
2823 }
else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2838 return GetUndefBooleanConstant();
2849 unsigned BitWidth =
Op.getScalarValueSizeInBits();
2858 unsigned Opc =
Op.getOpcode();
2867 return (NoFPClass & TestMask) == TestMask;
2874 return Op->getFlags().hasNoNaNs();
2900 unsigned Depth)
const {
2908 const APInt &DemandedElts,
2909 unsigned Depth)
const {
2916 unsigned Depth )
const {
2922 unsigned Depth)
const {
2927 const APInt &DemandedElts,
2928 unsigned Depth)
const {
2929 EVT VT =
Op.getValueType();
2936 for (
unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2937 if (!DemandedElts[EltIdx])
2941 KnownZeroElements.
setBit(EltIdx);
2943 return KnownZeroElements;
2953 unsigned Opcode = V.getOpcode();
2954 EVT VT = V.getValueType();
2957 "scalable demanded bits are ignored");
2969 UndefElts = V.getOperand(0).isUndef()
2978 APInt UndefLHS, UndefRHS;
2987 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
2988 UndefElts = UndefLHS | UndefRHS;
3002 return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, *
this,
3019 for (
unsigned i = 0; i != NumElts; ++i) {
3025 if (!DemandedElts[i])
3027 if (Scl && Scl !=
Op)
3038 for (
int i = 0; i != (int)NumElts; ++i) {
3044 if (!DemandedElts[i])
3046 if (M < (
int)NumElts)
3049 DemandedRHS.
setBit(M - NumElts);
3061 auto CheckSplatSrc = [&](
SDValue Src,
const APInt &SrcElts) {
3063 return (SrcElts.popcount() == 1) ||
3065 (SrcElts & SrcUndefs).
isZero());
3067 if (!DemandedLHS.
isZero())
3068 return CheckSplatSrc(V.getOperand(0), DemandedLHS);
3069 return CheckSplatSrc(V.getOperand(1), DemandedRHS);
3075 if (Src.getValueType().isScalableVector())
3077 uint64_t Idx = V.getConstantOperandVal(1);
3078 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3080 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts).
shl(Idx);
3082 UndefElts = UndefSrcElts.
extractBits(NumElts, Idx);
3093 if (Src.getValueType().isScalableVector())
3097 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts);
3099 UndefElts = UndefSrcElts.
trunc(NumElts);
3106 EVT SrcVT = Src.getValueType();
3116 if ((
BitWidth % SrcBitWidth) == 0) {
3118 unsigned Scale =
BitWidth / SrcBitWidth;
3120 APInt ScaledDemandedElts =
3122 for (
unsigned I = 0;
I != Scale; ++
I) {
3126 SubDemandedElts &= ScaledDemandedElts;
3130 if (!SubUndefElts.
isZero())
3144 EVT VT = V.getValueType();
3154 (AllowUndefs || !UndefElts);
3160 EVT VT = V.getValueType();
3161 unsigned Opcode = V.getOpcode();
3182 SplatIdx = (UndefElts & DemandedElts).
countr_one();
3197 if (!SVN->isSplat())
3199 int Idx = SVN->getSplatIndex();
3200 int NumElts = V.getValueType().getVectorNumElements();
3201 SplatIdx = Idx % NumElts;
3202 return V.getOperand(Idx / NumElts);
3214 if (LegalTypes && !TLI->isTypeLegal(SVT)) {
3217 LegalSVT = TLI->getTypeToTransformTo(*
getContext(), LegalSVT);
3218 if (LegalSVT.
bitsLT(SVT))
3226std::optional<ConstantRange>
3228 unsigned Depth)
const {
3231 "Unknown shift node");
3233 unsigned BitWidth = V.getScalarValueSizeInBits();
3236 const APInt &ShAmt = Cst->getAPIntValue();
3238 return std::nullopt;
3243 const APInt *MinAmt =
nullptr, *MaxAmt =
nullptr;
3244 for (
unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3245 if (!DemandedElts[i])
3249 MinAmt = MaxAmt =
nullptr;
3252 const APInt &ShAmt = SA->getAPIntValue();
3254 return std::nullopt;
3255 if (!MinAmt || MinAmt->
ugt(ShAmt))
3257 if (!MaxAmt || MaxAmt->ult(ShAmt))
3260 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3261 "Failed to find matching min/max shift amounts");
3262 if (MinAmt && MaxAmt)
3272 return std::nullopt;
3275std::optional<unsigned>
3277 unsigned Depth)
const {
3280 "Unknown shift node");
3281 if (std::optional<ConstantRange> AmtRange =
3283 if (
const APInt *ShAmt = AmtRange->getSingleElement())
3284 return ShAmt->getZExtValue();
3285 return std::nullopt;
3288std::optional<unsigned>
3294std::optional<unsigned>
3296 unsigned Depth)
const {
3299 "Unknown shift node");
3300 if (std::optional<ConstantRange> AmtRange =
3302 return AmtRange->getUnsignedMin().getZExtValue();
3303 return std::nullopt;
3306std::optional<unsigned>
3312std::optional<unsigned>
3314 unsigned Depth)
const {
3317 "Unknown shift node");
3318 if (std::optional<ConstantRange> AmtRange =
3320 return AmtRange->getUnsignedMax().getZExtValue();
3321 return std::nullopt;
3324std::optional<unsigned>
3342 unsigned Depth)
const {
3343 unsigned BitWidth =
Op.getScalarValueSizeInBits();
3347 if (
auto OptAPInt =
Op->bitcastToAPInt()) {
3349 APInt V = *std::move(OptAPInt);
3373 assert((!
Op.getValueType().isScalableVector() || NumElts == 1) &&
3374 "DemandedElts for scalable vectors must be 1 to represent all lanes");
3375 assert((!
Op.getValueType().isFixedLengthVector() ||
3376 NumElts ==
Op.getValueType().getVectorNumElements()) &&
3377 "Unexpected vector size");
3382 unsigned Opcode =
Op.getOpcode();
3396 "Expected SPLAT_VECTOR implicit truncation");
3403 unsigned ScalarSize =
Op.getOperand(0).getScalarValueSizeInBits();
3405 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3412 const APInt &Step =
Op.getConstantOperandAPInt(0);
3421 const APInt MinNumElts =
3427 .
umul_ov(MinNumElts, Overflow);
3431 const APInt MaxValue = (MaxNumElts - 1).
umul_ov(Step, Overflow);
3439 assert(!
Op.getValueType().isScalableVector());
3441 Known.setAllConflict();
3442 for (
unsigned i = 0, e =
Op.getNumOperands(); i != e; ++i) {
3443 if (!DemandedElts[i])
3455 "Expected BUILD_VECTOR implicit truncation");
3463 if (
Known.isUnknown())
3468 if (
Known.hasConflict())
3476 if (
Known.isUnknown())
3483 assert(!
Op.getValueType().isScalableVector());
3486 APInt DemandedLHS, DemandedRHS;
3490 DemandedLHS, DemandedRHS))
3494 Known.setAllConflict();
3495 if (!!DemandedLHS) {
3501 if (
Known.isUnknown())
3503 if (!!DemandedRHS) {
3512 const APInt &Multiplier =
Op.getConstantOperandAPInt(0);
3517 if (
Op.getValueType().isScalableVector())
3520 Known.setAllConflict();
3521 EVT SubVectorVT =
Op.getOperand(0).getValueType();
3523 unsigned NumSubVectors =
Op.getNumOperands();
3524 for (
unsigned i = 0; i != NumSubVectors; ++i) {
3526 DemandedElts.
extractBits(NumSubVectorElts, i * NumSubVectorElts);
3527 if (!!DemandedSub) {
3533 if (
Known.isUnknown())
3539 if (
Op.getValueType().isScalableVector())
3545 uint64_t Idx =
Op.getConstantOperandVal(2);
3546 unsigned NumSubElts =
Sub.getValueType().getVectorNumElements();
3548 APInt DemandedSrcElts = DemandedElts;
3549 DemandedSrcElts.
clearBits(Idx, Idx + NumSubElts);
3551 Known.setAllConflict();
3552 if (!!DemandedSubElts) {
3554 if (
Known.isUnknown())
3557 if (!!DemandedSrcElts) {
3567 APInt DemandedSrcElts;
3568 if (Src.getValueType().isScalableVector())
3569 DemandedSrcElts =
APInt(1, 1);
3571 uint64_t Idx =
Op.getConstantOperandVal(1);
3572 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3573 DemandedSrcElts = DemandedElts.
zext(NumSrcElts).
shl(Idx);
3579 if (
Op.getValueType().isScalableVector())
3583 if (DemandedElts != 1)
3594 if (
Op.getValueType().isScalableVector())
3614 if ((
BitWidth % SubBitWidth) == 0) {
3621 unsigned SubScale =
BitWidth / SubBitWidth;
3622 APInt SubDemandedElts(NumElts * SubScale, 0);
3623 for (
unsigned i = 0; i != NumElts; ++i)
3624 if (DemandedElts[i])
3625 SubDemandedElts.
setBit(i * SubScale);
3627 for (
unsigned i = 0; i != SubScale; ++i) {
3630 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3631 Known.insertBits(Known2, SubBitWidth * Shifts);
3636 if ((SubBitWidth %
BitWidth) == 0) {
3637 assert(
Op.getValueType().isVector() &&
"Expected bitcast to vector");
3642 unsigned SubScale = SubBitWidth /
BitWidth;
3643 APInt SubDemandedElts =
3647 Known.setAllConflict();
3648 for (
unsigned i = 0; i != NumElts; ++i)
3649 if (DemandedElts[i]) {
3650 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3654 if (
Known.isUnknown())
3681 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3692 if (
Op->getFlags().hasNoSignedWrap() &&
3693 Op.getOperand(0) ==
Op.getOperand(1) &&
3694 !
Known.isNegative())
3695 Known.makeNonNegative();
3720 unsigned SignBits1 =
3724 unsigned SignBits0 =
3726 Known.Zero.setHighBits(std::min(SignBits0, SignBits1) - 1);
3730 assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3733 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3734 if (
Op.getResNo() == 0)
3741 assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3744 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3745 if (
Op.getResNo() == 0)
3793 if (
Known.isUnknown())
3803 if (
Known.isUnknown())
3812 if (
Op.getResNo() != 1)
3818 if (TLI->getBooleanContents(
Op.getValueType().isVector(),
false) ==
3821 Known.Zero.setBitsFrom(1);
3827 unsigned OpNo =
Op->isStrictFPOpcode() ? 1 : 0;
3829 if (TLI->getBooleanContents(
Op.getOperand(OpNo).getValueType()) ==
3832 Known.Zero.setBitsFrom(1);
3839 bool NUW =
Op->getFlags().hasNoUnsignedWrap();
3840 bool NSW =
Op->getFlags().hasNoSignedWrap();
3847 if (std::optional<unsigned> ShMinAmt =
3849 Known.Zero.setLowBits(*ShMinAmt);
3856 Op->getFlags().hasExact());
3859 if (std::optional<unsigned> ShMinAmt =
3861 Known.Zero.setHighBits(*ShMinAmt);
3867 Op->getFlags().hasExact());
3873 unsigned Amt =
C->getAPIntValue().urem(
BitWidth);
3888 unsigned Amt =
C->getAPIntValue().urem(
BitWidth);
3894 DemandedElts,
Depth + 1);
3910 assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3913 unsigned LoBits =
Op.getOperand(0).getScalarValueSizeInBits();
3914 unsigned HiBits =
Op.getOperand(1).getScalarValueSizeInBits();
3931 if (
Op.getResNo() == 0)
3949 Known.Zero.setBitsFrom(LowBits);
3958 Known.Zero.setBitsFrom(LowBits);
3962 unsigned MinRedundantSignBits =
3978 Known.Zero.setBitsFrom(1);
4014 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
4019 !
Op.getValueType().isScalableVector()) {
4031 Known.setAllConflict();
4032 for (
unsigned i = 0; i != NumElts; ++i) {
4033 if (!DemandedElts[i])
4043 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4049 Known.One.clearAllBits();
4050 Known.Zero.clearAllBits();
4062 }
else if (
Op.getResNo() == 0) {
4063 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4064 KnownBits KnownScalarMemory(ScalarMemorySize);
4065 if (
const MDNode *MD = LD->getRanges())
4076 Known = KnownScalarMemory;
4083 if (
Op.getValueType().isScalableVector())
4085 EVT InVT =
Op.getOperand(0).getValueType();
4097 if (
Op.getValueType().isScalableVector())
4099 EVT InVT =
Op.getOperand(0).getValueType();
4115 if (
Op.getValueType().isScalableVector())
4117 EVT InVT =
Op.getOperand(0).getValueType();
4152 Known.Zero |= (~InMask);
4163 Known.Zero.setLowBits(LogOfAlign);
4164 Known.One.clearLowBits(LogOfAlign);
4173 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
4175 Known.makeNonNegative();
4179 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
4181 Known.makeNegative();
4189 Known.Zero.setSignBit();
4190 Known.One.clearSignBit();
4194 Known.Zero.setBitsFrom(1);
4200 bool SelfAdd =
Op.getOperand(0) ==
Op.getOperand(1) &&
4202 Op.getOperand(0), DemandedElts,
4205 Flags.hasNoUnsignedWrap(), SelfAdd);
4213 Flags.hasNoUnsignedWrap());
4220 if (
Op.getResNo() == 1) {
4222 if (TLI->getBooleanContents(
Op.getOperand(0).getValueType()) ==
4225 Known.Zero.setBitsFrom(1);
4231 "We only compute knownbits for the difference here.");
4238 Borrow = Borrow.
trunc(1);
4252 if (
Op.getResNo() == 1) {
4254 if (TLI->getBooleanContents(
Op.getOperand(0).getValueType()) ==
4257 Known.Zero.setBitsFrom(1);
4263 assert(
Op.getResNo() == 0 &&
"We only compute knownbits for the sum here.");
4273 Carry = Carry.
trunc(1);
4309 const unsigned Index =
Op.getConstantOperandVal(1);
4310 const unsigned EltBitWidth =
Op.getValueSizeInBits();
4312 Known =
Known.extractBits(EltBitWidth, Index * EltBitWidth);
4334 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4352 if (
Op.getValueType().isScalableVector())
4361 bool DemandedVal =
true;
4362 APInt DemandedVecElts = DemandedElts;
4364 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4365 unsigned EltIdx = CEltNo->getZExtValue();
4366 DemandedVal = !!DemandedElts[EltIdx];
4369 Known.setAllConflict();
4374 if (!!DemandedVecElts) {
4394 Known.Zero.setHighBits(
4426 if (CstLow && CstHigh) {
4431 const APInt &ValueHigh = CstHigh->getAPIntValue();
4432 if (ValueLow.
sle(ValueHigh)) {
4435 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
4437 Known.One.setHighBits(MinSignBits);
4441 Known.Zero.setHighBits(MinSignBits);
4458 if (IsMax && CstLow) {
4469 Known.makeNonNegative();
4475 Known.makeNonNegative();
4477 Known.makeNegative();
4488 if (
Op.getResNo() == 0) {
4490 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4491 KnownBits KnownScalarMemory(ScalarMemorySize);
4492 if (
const MDNode *MD = AT->getRanges())
4495 switch (AT->getExtensionType()) {
4503 switch (TLI->getExtendForAtomicOps()) {
4516 Known = KnownScalarMemory;
4524 if (
Op.getResNo() == 1) {
4529 if (TLI->getBooleanContents(
Op.getValueType().isVector(),
false) ==
4532 Known.Zero.setBitsFrom(1);
4550 if (
Op.getResNo() == 0) {
4552 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4555 Known.Zero.setBitsFrom(MemBits);
4563 TLI->computeKnownBitsForStackObjectPointer(
4564 Known, MF, MF.getFrameInfo().getObjectAlign(FrameIdx));
4567 case ISD::VP_LOAD_FF: {
4568 if (
Op.getResNo() != 1)
4575 EVT ResVT =
Op->getValueType(0);
4594 TLI->computeKnownBitsForTargetNode(
Op,
Known, DemandedElts, *
this,
Depth);
4726 unsigned Depth)
const {
4732 const APInt &DemandedElts,
4734 unsigned Depth)
const {
4735 EVT VT =
Op.getValueType();
4739 return ConstantRange::getFull(
BitWidth);
4744 unsigned Opcode =
Op.getOpcode();
4748 const APInt &Multiplier =
Op.getConstantOperandAPInt(0);
4755 return ConstantRange::getFull(
BitWidth);
4760 unsigned Depth)
const {
4768 unsigned Depth)
const {
4778 unsigned Depth)
const {
4784 const APInt &DemandedElts,
4785 bool OrZero,
unsigned Depth)
const {
4791 [[maybe_unused]]
unsigned NumElts = DemandedElts.
getBitWidth();
4793 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4796 "Unexpected vector size");
4800 return (OrZero && V.isZero()) || V.isPowerOf2();
4823 APInt DemandedSrcElts =
4824 ConstEltNo && ConstEltNo->getAPIntValue().
ult(NumSrcElts)
4849 if (
C &&
C->getAPIntValue() == 1)
4860 if (
C &&
C->getAPIntValue().isSignMask())
4910 APInt DemandedLHS, DemandedRHS;
4914 DemandedLHS, DemandedRHS))
4938 return C1->getValueAPF().getExactLog2Abs() >= 0;
4952 unsigned Depth)
const {
4953 EVT VT =
Op.getValueType();
4958 unsigned FirstAnswer = 1;
4961 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4964 const APInt &Val =
C->getAPIntValue();
4974 unsigned Opcode =
Op.getOpcode();
4979 return VTBits-Tmp+1;
4993 unsigned NumSrcBits =
Op.getOperand(0).getValueSizeInBits();
4995 if (NumSrcSignBits > (NumSrcBits - VTBits))
4996 return NumSrcSignBits - (NumSrcBits - VTBits);
5002 for (
unsigned i = 0, e =
Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
5003 if (!DemandedElts[i])
5010 APInt T =
C->getAPIntValue().trunc(VTBits);
5011 Tmp2 =
T.getNumSignBits();
5013 SrcOp.getOperand(0).getScalarValueSizeInBits() >= VTBits) {
5020 APInt DemandedSrcElts;
5023 DemandedSrcElts =
APInt(1, 1);
5027 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
5037 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
5041 if (
SrcOp.getValueSizeInBits() != VTBits) {
5043 "Expected BUILD_VECTOR implicit truncation");
5044 unsigned ExtraBits =
SrcOp.getValueSizeInBits() - VTBits;
5045 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
5048 Tmp = std::min(Tmp, Tmp2);
5059 Tmp = std::min(Tmp, Tmp2);
5066 APInt DemandedLHS, DemandedRHS;
5070 DemandedLHS, DemandedRHS))
5073 Tmp = std::numeric_limits<unsigned>::max();
5076 if (!!DemandedRHS) {
5078 Tmp = std::min(Tmp, Tmp2);
5083 assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
5099 if (VTBits == SrcBits)
5105 if ((SrcBits % VTBits) == 0) {
5108 unsigned Scale = SrcBits / VTBits;
5109 APInt SrcDemandedElts =
5119 for (
unsigned i = 0; i != NumElts; ++i)
5120 if (DemandedElts[i]) {
5121 unsigned SubOffset = i % Scale;
5122 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
5123 SubOffset = SubOffset * VTBits;
5124 if (Tmp <= SubOffset)
5126 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
5136 return VTBits - Tmp + 1;
5138 Tmp = VTBits -
Op.getOperand(0).getScalarValueSizeInBits();
5145 return std::max(Tmp, Tmp2);
5150 EVT SrcVT = Src.getValueType();
5158 if (std::optional<unsigned> ShAmt =
5160 Tmp = std::min(Tmp + *ShAmt, VTBits);
5163 if (std::optional<ConstantRange> ShAmtRange =
5165 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
5166 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
5177 unsigned SizeDifference =
5179 if (SizeDifference <= MinShAmt) {
5180 Tmp = SizeDifference +
5183 return Tmp - MaxShAmt;
5189 return Tmp - MaxShAmt;
5199 FirstAnswer = std::min(Tmp, Tmp2);
5209 if (Tmp == 1)
return 1;
5211 return std::min(Tmp, Tmp2);
5214 if (Tmp == 1)
return 1;
5216 return std::min(Tmp, Tmp2);
5228 if (CstLow && CstHigh) {
5233 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
5234 return std::min(Tmp, Tmp2);
5243 return std::min(Tmp, Tmp2);
5251 return std::min(Tmp, Tmp2);
5255 if (
Op.getResNo() == 0 &&
Op.getOperand(0) ==
Op.getOperand(1))
5266 if (
Op.getResNo() != 1)
5272 if (TLI->getBooleanContents(VT.
isVector(),
false) ==
5280 unsigned OpNo =
Op->isStrictFPOpcode() ? 1 : 0;
5282 if (TLI->getBooleanContents(
Op.getOperand(OpNo).getValueType()) ==
5290 if (TLI->getBooleanContents(VT.
isVector(),
false) ==
5299 Tmp, VTBits,
C ? std::optional(
C->getAPIntValue()) : std::nullopt,
5307 if (Tmp == 1)
return 1;
5312 if (CRHS->isAllOnes()) {
5318 if ((
Known.Zero | 1).isAllOnes())
5323 if (
Known.isNonNegative())
5328 if (Tmp2 == 1)
return 1;
5332 return std::min(Tmp, Tmp2) - 1;
5335 if (Tmp2 == 1)
return 1;
5340 if (CLHS->isZero()) {
5345 if ((
Known.Zero | 1).isAllOnes())
5350 if (
Known.isNonNegative())
5359 if (Tmp == 1)
return 1;
5360 return std::min(Tmp, Tmp2) - 1;
5363 unsigned SignBitsOp0 =
5365 if (SignBitsOp0 == 1)
5367 unsigned SignBitsOp1 =
5369 if (SignBitsOp1 == 1)
5371 unsigned OutValidBits =
5372 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5373 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5382 return std::min(Tmp, Tmp2);
5391 unsigned NumSrcBits =
Op.getOperand(0).getScalarValueSizeInBits();
5392 unsigned NumSrcSignBits =
5394 if (NumSrcSignBits > (NumSrcBits - VTBits))
5395 return NumSrcSignBits - (NumSrcBits - VTBits);
5402 const int BitWidth =
Op.getValueSizeInBits();
5403 const int Items =
Op.getOperand(0).getValueSizeInBits() /
BitWidth;
5407 const int rIndex = Items - 1 -
Op.getConstantOperandVal(1);
5422 bool DemandedVal =
true;
5423 APInt DemandedVecElts = DemandedElts;
5425 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
5426 unsigned EltIdx = CEltNo->getZExtValue();
5427 DemandedVal = !!DemandedElts[EltIdx];
5430 Tmp = std::numeric_limits<unsigned>::max();
5436 Tmp = std::min(Tmp, Tmp2);
5438 if (!!DemandedVecElts) {
5440 Tmp = std::min(Tmp, Tmp2);
5442 assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
5452 const unsigned BitWidth =
Op.getValueSizeInBits();
5453 const unsigned EltBitWidth =
Op.getOperand(0).getScalarValueSizeInBits();
5466 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
5476 APInt DemandedSrcElts;
5477 if (Src.getValueType().isScalableVector())
5478 DemandedSrcElts =
APInt(1, 1);
5480 uint64_t Idx =
Op.getConstantOperandVal(1);
5481 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5482 DemandedSrcElts = DemandedElts.
zext(NumSrcElts).
shl(Idx);
5491 Tmp = std::numeric_limits<unsigned>::max();
5492 EVT SubVectorVT =
Op.getOperand(0).getValueType();
5494 unsigned NumSubVectors =
Op.getNumOperands();
5495 for (
unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5497 DemandedElts.
extractBits(NumSubVectorElts, i * NumSubVectorElts);
5501 Tmp = std::min(Tmp, Tmp2);
5503 assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
5516 uint64_t Idx =
Op.getConstantOperandVal(2);
5517 unsigned NumSubElts =
Sub.getValueType().getVectorNumElements();
5519 APInt DemandedSrcElts = DemandedElts;
5520 DemandedSrcElts.
clearBits(Idx, Idx + NumSubElts);
5522 Tmp = std::numeric_limits<unsigned>::max();
5523 if (!!DemandedSubElts) {
5528 if (!!DemandedSrcElts) {
5530 Tmp = std::min(Tmp, Tmp2);
5532 assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
5537 if (
Op.getResNo() != 0)
5541 if (
const MDNode *Ranges = LD->getRanges()) {
5542 if (DemandedElts != 1)
5547 switch (LD->getExtensionType()) {
5565 unsigned ExtType = LD->getExtensionType();
5570 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5571 return VTBits - Tmp + 1;
5573 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5574 return VTBits - Tmp;
5576 if (
const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5579 Type *CstTy = Cst->getType();
5584 for (
unsigned i = 0; i != NumElts; ++i) {
5585 if (!DemandedElts[i])
5590 Tmp = std::min(Tmp,
Value.getNumSignBits());
5594 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5595 Tmp = std::min(Tmp,
Value.getNumSignBits());
5627 if (
Op.getResNo() == 0) {
5628 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5634 switch (AT->getExtensionType()) {
5638 return VTBits - Tmp + 1;
5640 return VTBits - Tmp;
5645 return VTBits - Tmp + 1;
5647 return VTBits - Tmp;
5662 TLI->ComputeNumSignBitsForTargetNode(
Op, DemandedElts, *
this,
Depth);
5664 FirstAnswer = std::max(FirstAnswer, NumBits);
5671 return std::max(FirstAnswer,
Known.countMinSignBits());
5675 unsigned Depth)
const {
5677 return Op.getScalarValueSizeInBits() - SignBits + 1;
5681 const APInt &DemandedElts,
5682 unsigned Depth)
const {
5684 return Op.getScalarValueSizeInBits() - SignBits + 1;
5689 unsigned Depth)
const {
5699 const APInt &DemandedElts,
5701 unsigned Depth)
const {
5702 unsigned Opcode =
Op.getOpcode();
5731 EVT SrcVT = Src.getValueType();
5732 EVT DstVT =
Op.getValueType();
5742 if (SrcEltBits == DstEltBits)
5746 if (SrcEltBits < DstEltBits) {
5747 if (DstEltBits % SrcEltBits != 0)
5750 assert(NumSrcElts == NumDstElts * (DstEltBits / SrcEltBits) &&
5751 "Unexpected vector bitcast");
5752 APInt DemandedSrcElts =
5758 if (SrcEltBits % DstEltBits != 0)
5761 assert(NumDstElts == NumSrcElts * (SrcEltBits / DstEltBits) &&
5762 "Unexpected vector bitcast");
5763 APInt DemandedSrcElts =
5772 for (
unsigned i = 0, e =
Op.getNumOperands(); i < e; ++i) {
5773 if (!DemandedElts[i])
5781 EVT VT =
Op.getValueType();
5785 EVT SubVT =
Op.getOperand(0).getValueType();
5787 for (
unsigned I = 0, E =
Op.getNumOperands();
I != E; ++
I) {
5788 APInt DemandedSubElts =
5790 if (!!DemandedSubElts &&
5800 if (Src.getValueType().isScalableVector())
5802 uint64_t Idx =
Op.getConstantOperandVal(1);
5803 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5804 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts).
shl(Idx);
5810 if (
Op.getValueType().isScalableVector())
5814 uint64_t Idx =
Op.getConstantOperandVal(2);
5815 unsigned NumSubElts =
Sub.getValueType().getVectorNumElements();
5817 APInt DemandedSrcElts = DemandedElts;
5818 DemandedSrcElts.
clearBits(Idx, Idx + NumSubElts);
5821 Sub, DemandedSubElts, Kind,
Depth + 1))
5824 Src, DemandedSrcElts, Kind,
Depth + 1))
5832 EVT SrcVT = Src.getValueType();
5836 IndexC->getZExtValue());
5851 if (DemandedElts[IndexC->getZExtValue()] &&
5854 APInt InVecDemandedElts = DemandedElts;
5855 InVecDemandedElts.
clearBit(IndexC->getZExtValue());
5856 if (!!InVecDemandedElts &&
5859 InVecDemandedElts, Kind,
Depth + 1))
5871 if (DemandedElts[0] &&
5891 APInt DemandedLHS, DemandedRHS;
5894 DemandedLHS, DemandedRHS,
5897 if (!DemandedLHS.
isZero() &&
5901 if (!DemandedRHS.
isZero() &&
5949 return isGuaranteedNotToBeUndefOrPoison(V, DemandedElts, Kind,
5960 if (
const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5961 EVT VT =
Op.getValueType();
5962 Type *CstTy = Cst->getType();
5964 auto IsSafe = [Kind](
const Constant *
C) ->
bool {
5967 return ::isGuaranteedNotToBeUndefOrPoison(
C);
5969 return ::isGuaranteedNotToBePoison(
C);
5971 return ::isGuaranteedNotToBeUndef(
C);
5978 for (
unsigned I = 0;
I != NumElts; ++
I) {
5979 if (!DemandedElts[
I])
6003 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
6004 Op, DemandedElts, *
this, Kind,
Depth);
6015 return isGuaranteedNotToBeUndefOrPoison(V, Kind, Depth + 1);
6021 unsigned Depth)
const {
6029 unsigned Depth)
const {
6030 if (ConsiderFlags &&
includesPoison(Kind) &&
Op->hasPoisonGeneratingFlags())
6033 unsigned Opcode =
Op.getOpcode();
6126 if (
Op.getOperand(0).getValueType().isInteger())
6133 unsigned CCOp = Opcode ==
ISD::SETCC ? 2 : 4;
6135 return (
unsigned)CCCode & 0x10U;
6195 EVT VecVT =
Op.getOperand(0).getValueType();
6206 for (
auto [Idx, Elt] :
enumerate(SVN->getMask()))
6207 if (Elt < 0 && DemandedElts[Idx])
6219 return TLI->canCreateUndefOrPoisonForTargetNode(
6220 Op, DemandedElts, *
this, Kind, ConsiderFlags,
Depth);
6229 unsigned Opcode =
Op.getOpcode();
6231 return Op->getFlags().hasDisjoint() ||
6245 unsigned Depth)
const {
6251 const APInt &DemandedElts,
6253 unsigned Depth)
const {
6265 EVT VT =
Op.getValueType();
6269 "Unexpected vector size");
6274 unsigned Opcode =
Op.getOpcode();
6282 InterestedClasses,
Depth + 1);
6289 for (
unsigned I = 0, E =
Op.getNumOperands();
I != E; ++
I) {
6290 if (!DemandedElts[
I])
6302 if (
Known.isUnknown())
6310 EVT SrcVT = Src.getValueType();
6336 EVT SrcVT =
Op.getOperand(0).getValueType();
6341 if (VTNumElts != SrcVTNumElts)
6350 InterestedClasses,
Depth + 1);
6356 InterestedClasses,
Depth + 1);
6358 InterestedClasses,
Depth + 1);
6359 Known.copysign(KnownSign);
6364 InterestedClasses,
Depth + 1);
6367 Known.setKnownFPClasses(
Known.getKnownFPClasses() & ~AssertedClasses);
6372 EVT SrcVT = Src.getValueType();
6374 unsigned Idx =
Op.getConstantOperandVal(1);
6390 unsigned Idx =
Op.getConstantOperandVal(2);
6394 APInt DemandedMask =
6396 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6399 if (!DemandedSrcElts.
isZero())
6401 InterestedClasses,
Depth + 1);
6402 if (!DemandedSubElts.
isZero()) {
6404 SubVector, DemandedSubElts, InterestedClasses,
Depth + 1);
6409 if (!
Known.isUnknown())
6419 Op.getOperand(2), DemandedElts, InterestedClasses,
Depth + 1);
6423 Op.getOperand(1), DemandedElts, InterestedClasses,
Depth + 1);
6430 TLI->computeKnownFPClassForTargetNode(
Op,
Known, DemandedElts, *
this,
6440 unsigned Depth)
const {
6446 bool SNaN,
unsigned Depth)
const {
6447 assert(!DemandedElts.
isZero() &&
"No demanded elements");
6450 if (
Op->getFlags().hasNoNaNs())
6456 unsigned Opcode =
Op.getOpcode();
6558 EVT SrcVT = Src.getValueType();
6562 Idx->getZExtValue());
6569 if (Src.getValueType().isFixedLengthVector()) {
6570 unsigned Idx =
Op.getConstantOperandVal(1);
6571 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
6572 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts).
shl(Idx);
6582 unsigned Idx =
Op.getConstantOperandVal(2);
6588 APInt DemandedMask =
6590 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6593 bool NeverNaN =
true;
6594 if (!DemandedSrcElts.
isZero())
6597 if (NeverNaN && !DemandedSubElts.
isZero())
6606 unsigned NumElts =
Op.getNumOperands();
6607 for (
unsigned I = 0;
I != NumElts; ++
I)
6608 if (DemandedElts[
I] &&
6627 return TLI->isKnownNeverNaNForTargetNode(
Op, DemandedElts, *
this, SNaN,
6635 return Known.isKnownNever(NanMask);
6644 const APInt &DemandedElts,
6645 unsigned Depth)
const {
6646 assert(!DemandedElts.
isZero() &&
"No demanded elements");
6647 EVT VT =
Op.getValueType();
6659 unsigned Depth)
const {
6663 EVT OpVT =
Op.getValueType();
6666 assert(!
Op.getValueType().isFloatingPoint() &&
6667 "Floating point types unsupported - use isKnownNeverLogicalZero");
6681 switch (
Op.getOpcode()) {
6700 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
6717 if (
Op->getFlags().hasNoSignedWrap() ||
Op->getFlags().hasNoUnsignedWrap())
6722 if (ValKnown.
One[0])
6734 if (
Op.getValueType().isScalableVector())
6742 APInt DemandedLHS, DemandedRHS;
6744 assert(NumElts == SVN->getMask().size() &&
"Unexpected vector size");
6746 DemandedLHS, DemandedRHS))
6749 return (!DemandedLHS ||
6808 if (
Op->getFlags().hasExact())
6826 if (
Op->getFlags().hasExact())
6831 if (
Op->getFlags().hasNoUnsignedWrap())
6849 if (
Op->getFlags().hasNoSignedWrap() ||
Op->getFlags().hasNoUnsignedWrap())
6860 const APInt &Multiplier =
Op.getConstantOperandAPInt(0);
6874 return !
C1->isNegative();
6876 switch (
Op.getOpcode()) {
6916 case ISD::VP_REDUCE_FMIN:
6917 case ISD::VP_REDUCE_FMAX:
6918 case ISD::VP_REDUCE_FMINIMUM:
6919 case ISD::VP_REDUCE_FMAXIMUM:
6927 assert(
Use.getValueType().isFloatingPoint());
6929 unsigned Opcode =
User->getOpcode();
6943 return OperandNo == 0;
6973 return Known.isKnownNever(~AlwaysBeatsZero);
6985 if (
Op->use_size() > 2)
6988 [&](
const SDUse &
Use) { return canIgnoreSignBitOfZero(Use); });
6993 if (
A ==
B)
return true;
6998 if (CA->isZero() && CB->isZero())
return true;
7033 NotOperand = NotOperand->getOperand(0);
7035 if (
Other == NotOperand)
7038 return NotOperand ==
Other->getOperand(0) ||
7039 NotOperand ==
Other->getOperand(1);
7045 A =
A->getOperand(0);
7048 B =
B->getOperand(0);
7051 return MatchNoCommonBitsPattern(
A->getOperand(0),
A->getOperand(1),
B) ||
7052 MatchNoCommonBitsPattern(
A->getOperand(1),
A->getOperand(0),
B);
7058 assert(
A.getValueType() ==
B.getValueType() &&
7059 "Values must have the same type");
7081 "BUILD_VECTOR cannot be used with scalable types");
7083 "Incorrect element count in BUILD_VECTOR!");
7086 bool AllPoison =
true;
7089 return Op.isUndef();
7095 bool IsIdentity =
true;
7096 for (
int i = 0; i !=
NumOps; ++i) {
7098 Ops[i].getOperand(0).getValueType() != VT ||
7099 (IdentitySrc &&
Ops[i].getOperand(0) != IdentitySrc) ||
7101 Ops[i].getConstantOperandAPInt(1) != i) {
7105 IdentitySrc =
Ops[i].getOperand(0);
7118 assert(!
Ops.empty() &&
"Can't concatenate an empty list of vectors!");
7121 return Ops[0].getValueType() ==
Op.getValueType();
7123 "Concatenation of vectors with inconsistent value types!");
7124 assert((
Ops[0].getValueType().getVectorElementCount() *
Ops.size()) ==
7126 "Incorrect element count in vector concatenation!");
7128 if (
Ops.size() == 1)
7132 bool AllPoison =
true;
7135 return Op.isUndef();
7143 bool IsIdentity =
true;
7144 for (
unsigned i = 0, e =
Ops.size(); i != e; ++i) {
7146 unsigned IdentityIndex = i *
Op.getValueType().getVectorMinNumElements();
7148 Op.getOperand(0).getValueType() != VT ||
7149 (IdentitySrc &&
Op.getOperand(0) != IdentitySrc) ||
7150 Op.getConstantOperandVal(1) != IdentityIndex) {
7154 assert((!IdentitySrc || IdentitySrc ==
Op.getOperand(0)) &&
7155 "Unexpected identity source vector for concat of extracts");
7156 IdentitySrc =
Op.getOperand(0);
7159 assert(IdentitySrc &&
"Failed to set source vector of extracts");
7175 EVT OpVT =
Op.getValueType();
7193 SVT = (SVT.
bitsLT(
Op.getValueType()) ?
Op.getValueType() : SVT);
7218 if (
SDNode *E = lookupNode(ID,
DL, InsertToken))
7221 auto *
N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
7222 CSEMap.insert(
N, InsertToken);
7234 Flags = Inserter->getFlags();
7235 return getNode(Opcode,
DL, VT, N1, Flags);
7298 "STEP_VECTOR can only be used with scalable types");
7301 "Unexpected step operand");
7322 "Invalid FP cast!");
7326 "Vector element count mismatch!");
7344 "Invalid SIGN_EXTEND!");
7346 "SIGN_EXTEND result type type should be vector iff the operand "
7351 "Vector element count mismatch!");
7374 unsigned NumSignExtBits =
7385 "Invalid ZERO_EXTEND!");
7387 "ZERO_EXTEND result type type should be vector iff the operand "
7392 "Vector element count mismatch!");
7430 "Invalid ANY_EXTEND!");
7432 "ANY_EXTEND result type type should be vector iff the operand "
7437 "Vector element count mismatch!");
7462 "Invalid TRUNCATE!");
7464 "TRUNCATE result type type should be vector iff the operand "
7469 "Vector element count mismatch!");
7496 assert(VT.
isVector() &&
"This DAG node is restricted to vector types.");
7498 "The input must be the same size or smaller than the result.");
7501 "The destination vector type must have fewer lanes than the input.");
7510 "Invalid ABS_MIN_POISON!");
7517 "BSWAP types must be a multiple of 16 bits!");
7531 "Cannot BITCAST between types of different sizes!");
7544 "Illegal SCALAR_TO_VECTOR node!");
7603 "VECTOR_REPEAT requires a fixed-length vector operand");
7605 "VECTOR_REPEAT requires a scalable vector result");
7608 "VECTOR_REPEAT operand and result element counts must match");
7622 "Wrong operand type!");
7629 if (VT != MVT::Glue) {
7632 if (
SDNode *E = lookupNode(ID,
DL, InsertToken)) {
7633 E->intersectFlagsWith(Flags);
7637 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
7639 createOperands(
N,
Ops);
7640 CSEMap.insert(
N, InsertToken);
7642 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
7643 createOperands(
N,
Ops);
7698 if (!C2.getBoolValue())
7702 if (!C2.getBoolValue())
7706 if (!C2.getBoolValue())
7710 if (!C2.getBoolValue())
7744 return std::nullopt;
7749 bool IsUndef1,
const APInt &C2,
7751 if (!(IsUndef1 || IsUndef2))
7759 return std::nullopt;
7767 if (!TLI->isOffsetFoldingLegal(GA))
7772 int64_t
Offset = C2->getSExtValue();
7792 assert(
Ops.size() == 2 &&
"Div/rem should have 2 operands");
7799 return V.isUndef() || isNullConstant(V);
7840 const APInt &Val =
C->getAPIntValue();
7844 C->isTargetOpcode(),
C->isOpaque());
7851 C->isTargetOpcode(),
C->isOpaque());
7856 C->isTargetOpcode(),
C->isOpaque());
7858 C->isTargetOpcode(),
C->isOpaque());
7890 C->isTargetOpcode(),
C->isOpaque());
7918 if (VT == MVT::f16 &&
C->getValueType(0) == MVT::i16)
7920 if (VT == MVT::f32 &&
C->getValueType(0) == MVT::i32)
7922 if (VT == MVT::f64 &&
C->getValueType(0) == MVT::i64)
7924 if (VT == MVT::f128 &&
C->getValueType(0) == MVT::i128)
7985 return getConstant(V.bitcastToAPInt().getZExtValue(),
DL, VT);
7988 if (VT == MVT::i16 &&
C->getValueType(0) == MVT::f16)
7989 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(),
DL,
7991 if (VT == MVT::i16 &&
C->getValueType(0) == MVT::bf16)
7992 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(),
DL,
7994 if (VT == MVT::i32 &&
C->getValueType(0) == MVT::f32)
7997 if (VT == MVT::i64 &&
C->getValueType(0) == MVT::f64)
7998 return getConstant(V.bitcastToAPInt().getZExtValue(),
DL, VT);
8025 "Expected vector reduction base opcode to be foldable");
8040 if (
C1->isOpaque() || C2->isOpaque())
8043 std::optional<APInt> FoldAttempt =
8044 FoldValue(Opcode,
C1->getAPIntValue(), C2->getAPIntValue());
8050 "Can't fold vectors ops with scalar operands");
8058 if (TLI->isCommutativeBinOp(Opcode))
8074 const APInt &Val =
C1->getAPIntValue();
8075 return SignExtendInReg(Val, VT);
8088 ScalarOps.
push_back(SignExtendInReg(Val, OpVT));
8096 SignExtendInReg(
Ops[0].getConstantOperandAPInt(0),
8107 if (
C1 && C2 && C3) {
8108 if (
C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
8110 const APInt &
V1 =
C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
8111 &
V3 = C3->getAPIntValue();
8127 if (
C1 && C2 && C3) {
8159 unsigned InputEltBits =
Ops[1].getScalarValueSizeInBits();
8161 unsigned NumInputElts =
Ops[1].getValueType().getVectorNumElements();
8165 for (
unsigned I = 0;
I != NumAccElts; ++
I) {
8172 if (!
C ||
C->isOpaque())
8174 Results[
I] =
C->getAPIntValue().trunc(AccEltBits);
8179 for (
unsigned I = 0;
I != NumInputElts; ++
I) {
8180 const unsigned AccIdx =
I % NumAccElts;
8185 PoisonElts.
set(AccIdx);
8191 if (!LHS || !RHS || LHS->isOpaque() || RHS->isOpaque())
8194 APInt LHSVal = LHS->getAPIntValue().
trunc(InputEltBits);
8195 APInt RHSVal = RHS->getAPIntValue().
trunc(InputEltBits);
8196 LHSVal = IsLHSSigned ? LHSVal.
sext(AccEltBits) : LHSVal.
zext(AccEltBits);
8197 RHSVal = IsRHSSigned ? RHSVal.
sext(AccEltBits) : RHSVal.
zext(AccEltBits);
8198 Results[AccIdx] += LHSVal * RHSVal;
8207 EVT LegalSVT = AccEltVT;
8209 LegalSVT = TLI->getTypeToTransformTo(*
getContext(), LegalSVT);
8210 if (LegalSVT.
bitsLT(AccEltVT))
8215 for (
unsigned I = 0;
I != NumAccElts; ++
I)
8227 Ops[0].getValueType() == VT &&
Ops[1].getValueType() == VT &&
8240 if (BV1->getConstantRawBits(IsLE, EltBits, RawBits1, UndefElts1) &&
8241 BV2->getConstantRawBits(IsLE, EltBits, RawBits2, UndefElts2)) {
8245 Opcode, RawBits1[
I], UndefElts1[
I], RawBits2[
I], UndefElts2[
I]);
8256 BVEltVT = BV1->getOperand(0).getValueType();
8259 BVEltVT = BV2->getOperand(0).getValueType();
8265 DstBits, RawBits, DstUndefs,
8268 for (
unsigned I = 0, E = DstBits.
size();
I != E; ++
I) {
8293 ?
Ops[0].getConstantOperandAPInt(0) * RHSVal
8294 :
Ops[0].getConstantOperandAPInt(0) << RHSVal;
8299 auto IsScalarOrSameVectorSize = [NumElts](
const SDValue &
Op) {
8300 return !
Op.getValueType().isVector() ||
8301 Op.getValueType().getVectorElementCount() == NumElts;
8304 auto IsBuildVectorSplatVectorOrUndef = [](
const SDValue &
Op) {
8330 LegalSVT = TLI->getTypeToTransformTo(*
getContext(), LegalSVT);
8342 for (
unsigned I = 0;
I != NumVectorElts;
I++) {
8345 EVT InSVT =
Op.getValueType().getScalarType();
8388 if (LegalSVT != SVT)
8389 ScalarResult =
getNode(ExtendCode,
DL, LegalSVT, ScalarResult);
8427 Result =
V1.sadd_ov(V2, Overflow);
8430 Result =
V1.uadd_ov(V2, Overflow);
8433 Result =
V1.ssub_ov(V2, Overflow);
8436 Result =
V1.usub_ov(V2, Overflow);
8439 Result =
V1.smul_ov(V2, Overflow);
8442 Result =
V1.umul_ov(V2, Overflow);
8475 if (
Ops.size() != 2)
8486 if (N1CFP && N2CFP) {
8537 if (N1C && N1C->getValueAPF().isNegZero() && N2.
isUndef())
8560 if (SrcEltVT == DstEltVT)
8568 if (SrcBitSize == DstBitSize) {
8573 if (
Op.getValueType() != SrcEltVT)
8616 for (
unsigned I = 0, E = RawBits.
size();
I != E; ++
I) {
8617 if (UndefElements[
I])
8638 ID.AddInteger(
A.value());
8641 if (
SDNode *E = lookupNode(ID,
DL, InsertToken))
8645 newSDNode<AssertAlignSDNode>(
DL.getIROrder(),
DL.getDebugLoc(), VTs,
A);
8646 createOperands(
N, {Val});
8648 CSEMap.insert(
N, InsertToken);
8660 Flags = Inserter->getFlags();
8661 return getNode(Opcode,
DL, VT, N1, N2, Flags);
8666 if (!TLI->isCommutativeBinOp(Opcode))
8675 if ((N1C && !N2C) || (N1CFP && !N2CFP))
8689 "Operand is DELETED_NODE!");
8705 N2.
getValueType() == MVT::Other &&
"Invalid token factor!");
8709 if (N1 == N2)
return N1;
8725 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
8727 N1.
getValueType() == VT &&
"Binary operator types must match!");
8730 if (N2CV && N2CV->
isZero())
8740 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
8742 N1.
getValueType() == VT &&
"Binary operator types must match!");
8752 if (N2CV && N2CV->
isZero())
8766 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
8768 N1.
getValueType() == VT &&
"Binary operator types must match!");
8771 if (N2CV && N2CV->
isZero())
8775 const APInt &N2CImm = N2C->getAPIntValue();
8789 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
8791 N1.
getValueType() == VT &&
"Binary operator types must match!");
8809 "Types of operands of UCMP/SCMP must match");
8811 "Operands and return type of must both be scalars or vectors");
8815 "Result and operands must have the same number of elements");
8823 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
8825 N1.
getValueType() == VT &&
"Binary operator types must match!");
8829 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
8831 N1.
getValueType() == VT &&
"Binary operator types must match!");
8837 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
8839 N1.
getValueType() == VT &&
"Binary operator types must match!");
8845 assert(VT.
isInteger() &&
"This operator does not apply to FP types!");
8847 N1.
getValueType() == VT &&
"Binary operator types must match!");
8858 N1.
getValueType() == VT &&
"Binary operator types must match!");
8866 "Invalid FCOPYSIGN!");
8871 const APInt &ShiftImm = N2C->getAPIntValue();
8885 "Shift operators return type must be the same as their first arg");
8887 "Shifts only work on integers");
8889 "Vector shift amounts must be in the same as their first arg");
8896 "Invalid use of small shift amount with oversized value!");
8903 if (N2CV && N2CV->
isZero())
8909 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
8915 "IS_FPCLASS is used for a non-floating type");
8930 "AssertNoFPClass is used for a non-floating type");
8935 "FPClassTest value too large");
8944 "Cannot *_EXTEND_INREG FP types");
8946 "AssertSExt/AssertZExt type should be the vector element type "
8947 "rather than the vector type!");
8956 "Cannot *_EXTEND_INREG FP types");
8958 "SIGN_EXTEND_INREG type should be vector iff the operand "
8962 "Vector element counts must match in SIGN_EXTEND_INREG");
8964 if (
EVT == VT)
return N1;
8972 "FP_TO_*INT_SAT type should be vector iff the operand type is "
8976 "Vector element counts must match in FP_TO_*INT_SAT");
8978 "Type to saturate to must be a scalar.");
8985 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
8986 element type of the vector.");
9008 N2C->getZExtValue() % Factor);
9017 "BUILD_VECTOR used for scalable vectors");
9040 if (N1Op2C && N2C) {
9070 assert(N2C && (
unsigned)N2C->getZExtValue() < 2 &&
"Bad EXTRACT_ELEMENT!");
9074 "Wrong types for EXTRACT_ELEMENT!");
9085 unsigned Shift = ElementSize * N2C->getZExtValue();
9086 const APInt &Val = N1C->getAPIntValue();
9093 "VECTOR_SHUFFLE_VAR result and source types don't match.");
9095 "VECTOR_SHUFFLE_VAR operands must be vectors.");
9097 "VECTOR_SHUFFLE_VAR mask must be a vector of integers.");
9099 "VECTOR_SHUFFLE_VAR mask has the wrong number of elements.");
9108 "Extract subvector VTs must be vectors!");
9110 "Extract subvector VTs must have the same element type!");
9112 "Cannot extract a scalable vector from a fixed length vector!");
9115 "Extract subvector must be from larger vector to smaller vector!");
9116 assert(N2C &&
"Extract subvector index must be a constant");
9120 "Extract subvector overflow!");
9121 assert(N2C->getAPIntValue().getBitWidth() ==
9123 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
9125 "Extract index is not a multiple of the output vector length");
9140 return N1.
getOperand(N2C->getZExtValue() / Factor);
9181 if (TLI->isCommutativeBinOp(Opcode)) {
9260 if (VT != MVT::Glue) {
9263 if (
SDNode *E = lookupNode(ID,
DL, InsertToken)) {
9264 E->intersectFlagsWith(Flags);
9268 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
9270 createOperands(
N,
Ops);
9271 CSEMap.insert(
N, InsertToken);
9273 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
9274 createOperands(
N,
Ops);
9287 Flags = Inserter->getFlags();
9288 return getNode(Opcode,
DL, VT, N1, N2, N3, Flags);
9297 "Operand is DELETED_NODE!");
9316 "SETCC operands must have the same type!");
9318 "SETCC type should be vector iff the operand type is vector!");
9321 "SETCC vector element counts must match!");
9345 "INSERT_VECTOR_ELT vector type mismatch");
9347 "INSERT_VECTOR_ELT scalar fp/int mismatch");
9350 "INSERT_VECTOR_ELT fp scalar type mismatch");
9353 "INSERT_VECTOR_ELT int scalar size mismatch");
9399 "Dest and insert subvector source types must match!");
9401 "Insert subvector VTs must be vectors!");
9403 "Insert subvector VTs must have the same element type!");
9405 "Cannot insert a scalable vector into a fixed length vector!");
9408 "Insert subvector must be from smaller vector to larger vector!");
9410 "Insert subvector index must be constant");
9414 "Insert subvector overflow!");
9417 "Constant index for INSERT_SUBVECTOR has an invalid size");
9465 assert(VT == VecVT &&
"Vector and result type don't match.");
9467 "All inputs must be vectors.");
9468 assert(VecVT == PassthruVT &&
"Vector and passthru types don't match.");
9470 "Vector and mask must have same number of elements.");
9485 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9486 "node to have the same type!");
9488 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
9489 "the same type as its result!");
9492 "Expected the element count of the second and third operands of the "
9493 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
9494 "element count of the first operand and the result!");
9496 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9497 "node to have an element type which is the same as or smaller than "
9498 "the element type of the first operand and result!");
9523 if (VT != MVT::Glue) {
9526 if (
SDNode *E = lookupNode(ID,
DL, InsertToken)) {
9527 E->intersectFlagsWith(Flags);
9531 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
9533 createOperands(
N,
Ops);
9534 CSEMap.insert(
N, InsertToken);
9536 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
9537 createOperands(
N,
Ops);
9557 Flags = Inserter->getFlags();
9558 return getNode(Opcode,
DL, VT, N1, N2, N3, N4, Flags);
9573 Flags = Inserter->getFlags();
9574 return getNode(Opcode,
DL, VT, N1, N2, N3, N4, N5, Flags);
9591 if (FI->getIndex() < 0)
9606 assert(
C->getAPIntValue().getBitWidth() == 8);
9611 return DAG.
getConstant(Val, dl, VT,
false, IsOpaque);
9616 assert(
Value.getValueType() == MVT::i8 &&
"memset with non-byte fill value?");
9632 if (VT !=
Value.getValueType())
9645 if (Slice.Array ==
nullptr) {
9654 unsigned NumVTBytes = NumVTBits / 8;
9655 unsigned NumBytes = std::min(NumVTBytes,
unsigned(Slice.Length));
9657 APInt Val(NumVTBits, 0);
9659 for (
unsigned i = 0; i != NumBytes; ++i)
9662 for (
unsigned i = 0; i != NumBytes; ++i)
9663 Val |= (
uint64_t)(
unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
9669 if (
TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
9686 if (TLI->shouldPreservePtrArith(this->getMachineFunction().getFunction(),
9701 else if (Src->isAnyAdd() &&
9705 SrcDelta = Src.getConstantOperandVal(1);
9711 SrcDelta +
G->getOffset());
9727 assert(OutLoadChains.
size() &&
"Missing loads in memcpy inlining");
9728 assert(OutStoreChains.
size() &&
"Missing stores in memcpy inlining");
9730 for (
unsigned i = From; i < To; ++i) {
9732 GluedLoadChains.
push_back(OutLoadChains[i]);
9739 for (
unsigned i = From; i < To; ++i) {
9742 ST->getBasePtr(), ST->getMemoryVT(),
9743 ST->getMemOperand());
9751 Align SrcAlign,
bool isVol,
bool AlwaysInline,
9755 const MDNode *SrcMemCacheHint) {
9768 std::vector<EVT> MemOps;
9769 bool DstAlignCanChange =
false;
9775 DstAlignCanChange =
true;
9780 bool isZeroConstant = CopyFromConstant && Slice.Array ==
nullptr;
9781 unsigned Limit = AlwaysInline ? ~0U :
TLI.getMaxStoresPerMemcpy(OptSize);
9782 const MemOp Op = isZeroConstant
9786 SrcAlign, isVol, CopyFromConstant);
9787 if (!
TLI.findOptimalMemOpLowering(
9792 if (DstAlignCanChange) {
9793 Type *Ty = MemOps[0].getTypeForEVT(
C);
9794 Align NewDstAlign =
DL.getABITypeAlign(Ty);
9800 if (!
TRI->hasStackRealignment(MF))
9802 NewDstAlign = std::min(NewDstAlign, *StackAlign);
9804 if (NewDstAlign > DstAlign) {
9808 DstAlign = NewDstAlign;
9818 BatchAA && SrcVal &&
9826 unsigned NumMemOps = MemOps.size();
9828 for (
unsigned i = 0; i != NumMemOps; ++i) {
9833 if (VTSize >
Size) {
9836 assert(i == NumMemOps-1 && i != 0);
9837 SrcOff -= VTSize -
Size;
9838 DstOff -= VTSize -
Size;
9841 if (CopyFromConstant &&
9849 if (SrcOff < Slice.Length) {
9851 SubSlice.
move(SrcOff);
9854 SubSlice.
Array =
nullptr;
9856 SubSlice.
Length = VTSize;
9859 if (
Value.getNode()) {
9864 MMOMetadata(NewAAInfo,
nullptr, DstMemCacheHint));
9869 if (!
Store.getNode()) {
9875 EVT NVT =
TLI.getTypeToTransformTo(
C, VT);
9878 bool isDereferenceable =
9881 if (isDereferenceable)
9891 MMOMetadata(NewAAInfo,
nullptr, SrcMemCacheHint));
9898 MMOMetadata(NewAAInfo,
nullptr, DstMemCacheHint));
9907 unsigned GluedLdStLimit =
9908 MaxLdStGlue == 0 ?
TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
9909 unsigned NumLdStInMemcpy = OutStoreChains.
size();
9911 if (NumLdStInMemcpy) {
9915 if ((GluedLdStLimit <= 1) ||
9918 for (
unsigned i = 0; i < NumLdStInMemcpy; ++i) {
9924 if (NumLdStInMemcpy <= GluedLdStLimit) {
9926 NumLdStInMemcpy, OutLoadChains,
9929 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
9930 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
9931 unsigned GlueIter = 0;
9934 if (RemainingLdStInMemcpy) {
9936 DAG, dl, OutChains, NumLdStInMemcpy - RemainingLdStInMemcpy,
9937 NumLdStInMemcpy, OutLoadChains, OutStoreChains);
9940 for (
unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
9941 unsigned IndexFrom = NumLdStInMemcpy - RemainingLdStInMemcpy -
9942 GlueIter - GluedLdStLimit;
9943 unsigned IndexTo = NumLdStInMemcpy - RemainingLdStInMemcpy - GlueIter;
9945 OutLoadChains, OutStoreChains);
9946 GlueIter += GluedLdStLimit;
9969 std::vector<EVT> MemOps;
9970 bool DstAlignCanChange =
false;
9976 DstAlignCanChange =
true;
9978 unsigned Limit = AlwaysInline ? ~0U :
TLI.getMaxStoresPerMemmove(OptSize);
9979 if (!
TLI.findOptimalMemOpLowering(
9986 if (DstAlignCanChange) {
9987 Type *Ty = MemOps[0].getTypeForEVT(
C);
9988 Align NewDstAlign =
DL.getABITypeAlign(Ty);
9994 if (!
TRI->hasStackRealignment(MF))
9996 NewDstAlign = std::min(NewDstAlign, *StackAlign);
9998 if (NewDstAlign > DstAlign) {
10002 DstAlign = NewDstAlign;
10016 unsigned NumMemOps = MemOps.size();
10017 for (
unsigned i = 0; i < NumMemOps; i++) {
10018 EVT VT = MemOps[i];
10021 bool IsOverlapping =
false;
10023 if (i == NumMemOps - 1 && i != 0 && VTSize >
Size - SrcOff) {
10026 SrcOff =
Size - VTSize;
10027 IsOverlapping =
true;
10034 if (IsOverlapping) {
10038 if (!
TLI.allowsMisalignedMemoryAccesses(VT, SrcPtrInfo.
getAddrSpace(),
10039 SrcAlignAtOffset, MMOFlags,
10048 bool isDereferenceable =
10051 if (isDereferenceable)
10057 SrcMMOFlags, NewAAInfo);
10065 for (
unsigned i = 0; i < NumMemOps; i++) {
10066 EVT VT = MemOps[i];
10069 bool IsOverlapping =
false;
10071 if (i == NumMemOps - 1 && i != 0 && VTSize >
Size - DstOff) {
10074 DstOff =
Size - VTSize;
10075 IsOverlapping =
true;
10082 if (IsOverlapping) {
10086 if (!
TLI.allowsMisalignedMemoryAccesses(VT, DstPtrInfo.
getAddrSpace(),
10087 DstAlignAtOffset, MMOFlags,
10096 Chain, dl, LoadValues[i],
10098 DstPtrInfo.
getWithOffset(DstOff), DstAlignAtOffset, MMOFlags,
10139 std::vector<EVT> MemOps;
10140 bool DstAlignCanChange =
false;
10147 DstAlignCanChange =
true;
10149 unsigned Limit = AlwaysInline ? ~0 :
TLI.getMaxStoresPerMemset(OptSize);
10152 if (!
TLI.findOptimalMemOpLowering(
10154 MemOp::Set(
Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),
10159 if (DstAlignCanChange) {
10162 Align NewAlign =
DL.getABITypeAlign(Ty);
10168 if (!
TRI->hasStackRealignment(MF))
10170 NewAlign = std::min(NewAlign, *StackAlign);
10172 if (NewAlign > Alignment) {
10176 Alignment = NewAlign;
10182 unsigned NumMemOps = MemOps.size();
10187 LargestVT = MemOps[0];
10188 for (
unsigned i = 1; i < NumMemOps; i++)
10189 if (MemOps[i].bitsGT(LargestVT))
10190 LargestVT = MemOps[i];
10198 for (
unsigned i = 0; i < NumMemOps; i++) {
10199 EVT VT = MemOps[i];
10203 assert(
Size > 0 &&
"Target specified more stores than needed in "
10204 "findOptimalMemOpLowering");
10205 if (VTSize >
Size) {
10208 assert(i == NumMemOps-1 && i != 0);
10209 DstOff -= VTSize -
Size;
10216 if (VT.
bitsLT(LargestVT)) {
10221 TLI.isTruncateFree(LargestVT, VT))
10224 TLI.shallExtractConstSplatVectorElementToStore(
10227 TLI.isTypeLegal(SVT) &&
10236 assert(
Value.getValueType() == VT &&
"Value with wrong type.");
10247 if (VTSize >
Size) {
10256 assert(
Size == 0 &&
"Target's findOptimalMemOpLowering did not specify "
10257 "stores that exactly cover the memset size");
10266 if (AS != 0 && !
TLI->getTargetMachine().isNoopAddrSpaceCast(
DL, AS, 0)) {
10286 bool AllowReturnsFirstArg) {
10292 AllowReturnsFirstArg &&
10296static std::pair<SDValue, SDValue>
10303 if (LCImpl == RTLIB::Unsupported)
10318 CI->
getType(), Callee, std::move(Args))
10321 return TLI->LowerCallTo(CLI);
10331 RTLIB::STRCMP,
this, TLI);
10341 RTLIB::STRSTR,
this, TLI);
10357 RTLIB::MEMCCPY,
this, TLI);
10360std::pair<SDValue, SDValue>
10369 RTLIB::MEMCMP,
this, TLI);
10379 RTLIB::STRCPY,
this, TLI);
10390 RTLIB::STRLEN,
this, TLI);
10394 return TLI->supportSwiftError() &&
10395 MF->getFunction().getAttributes().hasAttrSomewhere(
10396 Attribute::SwiftError);
10401 Align DstAlign,
Align SrcAlign,
bool isVol,
bool AlwaysInline,
10402 const CallInst *CI, std::optional<bool> OverrideTailCall,
10409 const MDNode *DstMemCacheHint =
10411 const MDNode *SrcMemCacheHint =
10415 if (ConstantSize) {
10417 if (ConstantSize->
isZero())
10421 *
this, dl, Chain, Dst, Src, ConstantSize->
getZExtValue(), DstAlign,
10422 SrcAlign, isVol,
false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10423 DstMemCacheHint, SrcMemCacheHint);
10424 if (Result.getNode())
10432 CI,
this,
false)) &&
10435 SDValue Result = TSI->EmitTargetCodeForMemcpy(
10436 *
this, dl, Chain, Dst, Src,
Size, DstAlign, SrcAlign, isVol,
10437 AlwaysInline, IsTailCall, DstPtrInfo, SrcPtrInfo);
10438 assert((Result.getNode() || !IsTailCall ||
getRoot() != OldRoot) &&
10439 "target declined without clearing IsTailCall");
10440 if (Result.getNode() || IsTailCall)
10446 if (AlwaysInline) {
10447 assert(ConstantSize &&
"AlwaysInline requires a constant size!");
10449 *
this, dl, Chain, Dst, Src, ConstantSize->
getZExtValue(), DstAlign,
10450 SrcAlign, isVol,
true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA,
10451 DstMemCacheHint, SrcMemCacheHint);
10468 Args.emplace_back(Dst, PtrTy);
10469 Args.emplace_back(Src, PtrTy);
10473 bool IsTailCall =
false;
10474 RTLIB::LibcallImpl MemCpyImpl = Libcalls->getMemcpyImpl();
10476 if (OverrideTailCall.has_value()) {
10477 IsTailCall = *OverrideTailCall;
10479 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;
10489 Libcalls->getLibcallImplCallingConv(MemCpyImpl),
10490 Dst.getValueType().getTypeForEVT(*
getContext()),
10496 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10497 return CallResult.second;
10510 Type *SizeTy,
unsigned ElemSz,
10521 Args.emplace_back(Dst, ArgTy);
10522 Args.emplace_back(Src, ArgTy);
10525 RTLIB::Libcall LibraryCall =
10527 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(LibraryCall);
10528 if (LibcallImpl == RTLIB::Unsupported)
10535 Libcalls->getLibcallImplCallingConv(LibcallImpl),
10542 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10543 return CallResult.second;
10549 std::optional<bool> OverrideTailCall,
10559 if (ConstantSize) {
10561 if (ConstantSize->
isZero())
10565 *
this, dl, Chain, Dst, Src, ConstantSize->
getZExtValue(), DstAlign,
10566 SrcAlign, isVol,
false, DstPtrInfo, SrcPtrInfo, AAInfo);
10567 if (Result.getNode())
10575 CI,
this,
false)) &&
10578 SDValue Result = TSI->EmitTargetCodeForMemmove(
10579 *
this, dl, Chain, Dst, Src,
Size, DstAlign, SrcAlign, isVol, IsTailCall,
10580 DstPtrInfo, SrcPtrInfo);
10581 assert((Result.getNode() || !IsTailCall ||
getRoot() != OldRoot) &&
10582 "target declined without clearing IsTailCall");
10583 if (Result.getNode() || IsTailCall)
10598 Args.emplace_back(Dst, PtrTy);
10599 Args.emplace_back(Src, PtrTy);
10604 RTLIB::LibcallImpl MemmoveImpl = Libcalls->getLibcallImpl(RTLIB::MEMMOVE);
10606 bool IsTailCall =
false;
10607 if (OverrideTailCall.has_value()) {
10608 IsTailCall = *OverrideTailCall;
10610 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;
10620 Libcalls->getLibcallImplCallingConv(MemmoveImpl),
10621 Dst.getValueType().getTypeForEVT(*
getContext()),
10627 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10628 return CallResult.second;
10633 Type *SizeTy,
unsigned ElemSz,
10648 RTLIB::Libcall LibraryCall =
10650 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(LibraryCall);
10651 if (LibcallImpl == RTLIB::Unsupported)
10658 Libcalls->getLibcallImplCallingConv(LibcallImpl),
10665 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10666 return CallResult.second;
10671 bool isVol,
bool AlwaysInline,
10680 if (ConstantSize) {
10682 if (ConstantSize->
isZero())
10687 isVol,
false, DstPtrInfo, AAInfo);
10689 if (Result.getNode())
10700 SDValue Result = TSI->EmitTargetCodeForMemset(
10701 *
this, dl, Chain, Dst, Src,
Size, Alignment, isVol, AlwaysInline,
10702 IsTailCall, DstPtrInfo);
10703 assert((Result.getNode() || !IsTailCall ||
getRoot() != OldRoot) &&
10704 "target declined without clearing IsTailCall");
10705 if (Result.getNode() || IsTailCall)
10711 if (AlwaysInline) {
10712 assert(ConstantSize &&
"AlwaysInline requires a constant size!");
10715 isVol,
true, DstPtrInfo, AAInfo);
10717 "getMemsetStores must return a valid sequence when AlwaysInline");
10732 RTLIB::LibcallImpl BzeroImpl = Libcalls->getLibcallImpl(RTLIB::BZERO);
10733 bool UseBZero = BzeroImpl != RTLIB::Unsupported &&
isNullConstant(Src);
10739 Args.emplace_back(
Size,
DL.getIntPtrType(Ctx));
10741 Libcalls->getLibcallImplCallingConv(BzeroImpl),
Type::getVoidTy(Ctx),
10744 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(RTLIB::MEMSET);
10751 LibInfo->getExtAttrForI32Param());
10752 Args.emplace_back(
Size,
DL.getIntPtrType(Ctx));
10754 CLI.
setLibCallee(Libcalls->getLibcallImplCallingConv(MemsetImpl),
10755 Dst.getValueType().getTypeForEVT(Ctx),
10760 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(RTLIB::MEMSET);
10761 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;
10769 ReturnsFirstArg && LowersToMemset) &&
10775 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10776 return CallResult.second;
10781 Type *SizeTy,
unsigned ElemSz,
10794 LibInfo->getExtAttrForI32Param());
10797 RTLIB::Libcall LibraryCall =
10799 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(LibraryCall);
10800 if (LibcallImpl == RTLIB::Unsupported)
10807 Libcalls->getLibcallImplCallingConv(LibcallImpl),
10814 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10815 return CallResult.second;
10824 ID.AddInteger(getSyntheticNodeSubclassData<AtomicSDNode>(
10825 dl.
getIROrder(), Opcode, VTList, MemVT, MMO, ExtType));
10830 E->refineAlignment(MMO);
10831 E->refineMMOMetadata(MMO);
10836 VTList, MemVT, MMO, ExtType);
10837 createOperands(
N,
Ops);
10839 CSEMap.insert(
N, InsertToken);
10878 "Invalid Atomic Op");
10898 if (
Ops.size() == 1)
10912 for (
EVT VT : ResultTypes)
10922 if (
Size.hasValue() && !
Size.getValue())
10927 MF.getMachineMemOperand(PtrInfo, Flags,
Size, Alignment, AAInfo);
10943 assert(!MMOs.
empty() &&
"Must have at least one MMO");
10947 (Opcode <= (
unsigned)std::numeric_limits<int>::max() &&
10949 "Opcode is not a memory-accessing opcode!");
10952 if (MMOs.
size() == 1) {
10958 void *Buffer = Allocator.Allocate(AllocSize,
alignof(
size_t));
10959 size_t *CountPtr =
static_cast<size_t *
>(Buffer);
10960 *CountPtr = MMOs.
size();
10969 if (VTList.
VTs[VTList.
NumVTs-1] != MVT::Glue) {
10971 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
10972 Opcode, dl.
getIROrder(), VTList, MemVT, MemRefs));
10975 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
10976 ID.AddInteger(MMO->getFlags());
10979 if (
SDNode *E = lookupNode(ID, dl, InsertToken)) {
10985 VTList, MemVT, MemRefs);
10986 createOperands(
N,
Ops);
10987 CSEMap.insert(
N, InsertToken);
10990 VTList, MemVT, MemRefs);
10991 createOperands(
N,
Ops);
11000 SDValue Chain,
int FrameIndex) {
11002 const auto VTs =
getVTList(MVT::Other);
11011 if (
SDNode *E = lookupNode(ID, dl, InsertToken))
11016 createOperands(
N,
Ops);
11017 CSEMap.insert(
N, InsertToken);
11025 uint64_t
Guid, uint64_t Index,
11028 const auto VTs =
getVTList(MVT::Other);
11031 ID.AddInteger(
Guid);
11032 ID.AddInteger(Index);
11033 ID.AddInteger(Attr);
11035 if (
SDNode *E = lookupNode(ID, Dl, InsertToken))
11038 auto *
N = newSDNode<PseudoProbeSDNode>(
11040 createOperands(
N,
Ops);
11041 CSEMap.insert(
N, InsertToken);
11058 FI->getIndex(),
Offset);
11095 "Invalid chain type");
11107 MF.getMachineMemOperand(PtrInfo, MMOFlags,
Size, Alignment,
Metadata);
11108 return getLoad(AM, ExtType, VT, dl, Chain, Ptr,
Offset, MemVT, MMO);
11118 assert(VT == MemVT &&
"Non-extending load from different memory type!");
11122 "Should only be an extending load, not truncating!");
11124 "Cannot convert from FP to Int or Int -> FP!");
11126 "Cannot use an ext load to convert to or from a vector!");
11129 "Cannot use an ext load to change the number of vector elements!");
11136 "Range metadata and load type must match!");
11140 "Unindexed load with an offset!");
11147 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
11148 dl.
getIROrder(), VTs, AM, ExtType, MemVT, MMO));
11153 E->refineAlignment(MMO);
11154 E->refineMMOMetadata(MMO);
11158 ExtType, MemVT, MMO);
11159 createOperands(
N,
Ops);
11161 CSEMap.insert(
N, InsertToken);
11175 PtrInfo, VT, Alignment, MMOFlags,
Metadata);
11193 MemVT, Alignment, MMOFlags,
Metadata);
11209 "Load is already a indexed load!");
11212 LD->getMemOperand()->getFlags() &
11215 AM, LD->getExtensionType(), OrigLoad.
getValueType(), dl, LD->getChain(),
11216 Base,
Offset, LD->getPointerInfo(), LD->getMemoryVT(), LD->getAlign(),
11218 MMOMetadata(LD->getAAInfo(), LD->getRanges(), LD->getMemCacheHint()));
11230 assert(!
Metadata.Ranges &&
"range metadata is invalid for stores");
11238 MF.getMachineMemOperand(PtrInfo, MMOFlags,
Size, Alignment,
Metadata);
11239 return getStore(Chain, dl, Val, Ptr, MMO);
11252 bool IsTruncating) {
11256 IsTruncating =
false;
11257 }
else if (!IsTruncating) {
11258 assert(VT == SVT &&
"No-truncating store from different memory type!");
11261 "Should only be a truncating store, not extending!");
11264 "Cannot use trunc store to convert to or from a vector!");
11267 "Cannot use trunc store to change the number of vector elements!");
11272 "Unindexed store with an offset!");
11278 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
11279 dl.
getIROrder(), VTs, AM, IsTruncating, SVT, MMO));
11283 if (
SDNode *E = lookupNode(ID, dl, InsertToken)) {
11289 IsTruncating, SVT, MMO);
11290 createOperands(
N,
Ops);
11292 CSEMap.insert(
N, InsertToken);
11306 "Invalid chain type");
11310 assert(!
Metadata.Ranges &&
"range metadata is invalid for stores");
11327 PtrInfo, SVT, Alignment, MMOFlags,
Metadata);
11348 "Store is already a indexed store!");
11350 ST->getMemoryVT(), ST->getMemOperand(), AM,
11351 ST->isTruncatingStore());
11359 const MDNode *Ranges,
bool IsExpanding) {
11371 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr,
Offset, Mask, EVL, MemVT,
11380 bool IsExpanding) {
11382 assert(Mask.getValueType().getVectorElementCount() ==
11384 "Vector width mismatch between mask and data");
11388 "Unindexed load with an offset!");
11395 ID.AddInteger(getSyntheticNodeSubclassData<VPLoadSDNode>(
11396 dl.
getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
11401 E->refineAlignment(MMO);
11402 E->refineMMOMetadata(MMO);
11406 ExtType, IsExpanding, MemVT, MMO);
11407 createOperands(
N,
Ops);
11409 CSEMap.insert(
N, InsertToken);
11422 bool IsExpanding) {
11425 Mask, EVL, PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges,
11434 Mask, EVL, VT, MMO, IsExpanding);
11443 const AAMDNodes &AAInfo,
bool IsExpanding) {
11446 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo,
nullptr,
11456 EVL, MemVT, MMO, IsExpanding);
11463 bool IsCompressing) {
11465 assert(Mask.getValueType().getVectorElementCount() ==
11467 "Vector width mismatch between mask and data");
11471 "Unindexed vp_store with an offset!");
11477 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
11478 dl.
getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
11482 if (
SDNode *E = lookupNode(ID, dl, InsertToken)) {
11487 IsTruncating, IsCompressing, MemVT, MMO);
11488 createOperands(
N,
Ops);
11490 CSEMap.insert(
N, InsertToken);
11503 bool IsCompressing) {
11514 PtrInfo, MMOFlags, SVT.
getStoreSize(), Alignment, AAInfo);
11523 bool IsCompressing) {
11530 false, IsCompressing);
11533 "Should only be a truncating store, not extending!");
11536 "Cannot use trunc store to convert to or from a vector!");
11539 "Cannot use trunc store to change the number of vector elements!");
11546 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
11551 if (
SDNode *E = lookupNode(ID, dl, InsertToken)) {
11558 createOperands(
N,
Ops);
11560 CSEMap.insert(
N, InsertToken);
11573 "Unindexed load with an offset!");
11578 SDNodeKey ID(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs,
Ops);
11580 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
11581 DL.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
11585 if (
SDNode *E = lookupNode(ID,
DL, InsertToken)) {
11591 newSDNode<VPStridedLoadSDNode>(
DL.getIROrder(),
DL.getDebugLoc(), VTs, AM,
11592 ExtType, IsExpanding, MemVT, MMO);
11593 createOperands(
N,
Ops);
11594 CSEMap.insert(
N, InsertToken);
11605 bool IsExpanding) {
11608 Undef, Stride, Mask, EVL, VT, MMO, IsExpanding);
11617 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
11626 bool IsTruncating,
bool IsCompressing) {
11630 "Unindexed vp_store with an offset!");
11634 SDNodeKey ID(ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs,
Ops);
11636 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
11637 DL.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
11640 if (
SDNode *E = lookupNode(ID,
DL, InsertToken)) {
11644 auto *
N = newSDNode<VPStridedStoreSDNode>(
DL.getIROrder(),
DL.getDebugLoc(),
11645 VTs, AM, IsTruncating,
11646 IsCompressing, MemVT, MMO);
11647 createOperands(
N,
Ops);
11649 CSEMap.insert(
N, InsertToken);
11659 assert(
Ops.size() == 6 &&
"Incompatible number of operands");
11663 ID.AddInteger(getSyntheticNodeSubclassData<VPGatherSDNode>(
11668 if (
SDNode *E = lookupNode(ID, dl, InsertToken)) {
11674 VT, MMO, IndexType);
11675 createOperands(
N,
Ops);
11677 assert(
N->getMask().getValueType().getVectorElementCount() ==
11678 N->getValueType(0).getVectorElementCount() &&
11679 "Vector width mismatch between mask and data");
11680 assert(
N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11681 N->getValueType(0).getVectorElementCount().isScalable() &&
11682 "Scalable flags of index and data do not match");
11684 N->getIndex().getValueType().getVectorElementCount(),
11685 N->getValueType(0).getVectorElementCount()) &&
11686 "Vector width mismatch between index and data");
11688 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11689 "Scale should be a constant power of 2");
11691 CSEMap.insert(
N, InsertToken);
11702 assert(
Ops.size() == 7 &&
"Incompatible number of operands");
11706 ID.AddInteger(getSyntheticNodeSubclassData<VPScatterSDNode>(
11711 if (
SDNode *E = lookupNode(ID, dl, InsertToken)) {
11716 VT, MMO, IndexType);
11717 createOperands(
N,
Ops);
11719 assert(
N->getMask().getValueType().getVectorElementCount() ==
11720 N->getValue().getValueType().getVectorElementCount() &&
11721 "Vector width mismatch between mask and data");
11723 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11724 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11725 "Scalable flags of index and data do not match");
11727 N->getIndex().getValueType().getVectorElementCount(),
11728 N->getValue().getValueType().getVectorElementCount()) &&
11729 "Vector width mismatch between index and data");
11731 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11732 "Scale should be a constant power of 2");
11734 CSEMap.insert(
N, InsertToken);
11749 "Unindexed masked load with an offset!");
11755 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
11756 dl.
getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));
11760 if (
SDNode *E = lookupNode(ID, dl, InsertToken)) {
11765 AM, ExtTy, isExpanding, MemVT, MMO);
11766 createOperands(
N,
Ops);
11768 CSEMap.insert(
N, InsertToken);
11780 "Masked load is already a indexed load!");
11782 Offset, LD->getMask(), LD->getPassThru(),
11783 LD->getMemoryVT(), LD->getMemOperand(), AM,
11784 LD->getExtensionType(), LD->isExpandingLoad());
11792 bool IsCompressing) {
11794 "Invalid chain type");
11797 "Unindexed masked store with an offset!");
11803 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
11804 dl.
getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
11808 if (
SDNode *E = lookupNode(ID, dl, InsertToken)) {
11814 IsTruncating, IsCompressing, MemVT, MMO);
11815 createOperands(
N,
Ops);
11817 CSEMap.insert(
N, InsertToken);
11829 "Masked store is already a indexed store!");
11831 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
11832 AM, ST->isTruncatingStore(), ST->isCompressingStore());
11840 assert(
Ops.size() == 6 &&
"Incompatible number of operands");
11844 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
11845 dl.
getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));
11849 if (
SDNode *E = lookupNode(ID, dl, InsertToken)) {
11855 VTs, MemVT, MMO, IndexType, ExtTy);
11856 createOperands(
N,
Ops);
11858 assert(
N->getPassThru().getValueType() ==
N->getValueType(0) &&
11859 "Incompatible type of the PassThru value in MaskedGatherSDNode");
11860 assert(
N->getMask().getValueType().getVectorElementCount() ==
11861 N->getValueType(0).getVectorElementCount() &&
11862 "Vector width mismatch between mask and data");
11863 assert(
N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11864 N->getValueType(0).getVectorElementCount().isScalable() &&
11865 "Scalable flags of index and data do not match");
11867 N->getIndex().getValueType().getVectorElementCount(),
11868 N->getValueType(0).getVectorElementCount()) &&
11869 "Vector width mismatch between index and data");
11871 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11872 "Scale should be a constant power of 2");
11874 CSEMap.insert(
N, InsertToken);
11886 assert(
Ops.size() == 6 &&
"Incompatible number of operands");
11890 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
11891 dl.
getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));
11895 if (
SDNode *E = lookupNode(ID, dl, InsertToken)) {
11901 VTs, MemVT, MMO, IndexType, IsTrunc);
11902 createOperands(
N,
Ops);
11904 assert(
N->getMask().getValueType().getVectorElementCount() ==
11905 N->getValue().getValueType().getVectorElementCount() &&
11906 "Vector width mismatch between mask and data");
11908 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11909 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11910 "Scalable flags of index and data do not match");
11912 N->getIndex().getValueType().getVectorElementCount(),
11913 N->getValue().getValueType().getVectorElementCount()) &&
11914 "Vector width mismatch between index and data");
11916 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11917 "Scale should be a constant power of 2");
11919 CSEMap.insert(
N, InsertToken);
11930 assert(
Ops.size() == 7 &&
"Incompatible number of operands");
11934 ID.AddInteger(getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
11935 dl.
getIROrder(), VTs, MemVT, MMO, IndexType));
11939 if (
SDNode *E = lookupNode(ID, dl, InsertToken)) {
11945 VTs, MemVT, MMO, IndexType);
11946 createOperands(
N,
Ops);
11948 assert(
N->getMask().getValueType().getVectorElementCount() ==
11949 N->getIndex().getValueType().getVectorElementCount() &&
11950 "Vector width mismatch between mask and data");
11952 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11953 "Scale should be a constant power of 2");
11954 assert(
N->getInc().getValueType().isInteger() &&
"Non integer update value");
11956 CSEMap.insert(
N, InsertToken);
11970 ID.AddInteger(getSyntheticNodeSubclassData<VPLoadFFSDNode>(
DL.getIROrder(),
11975 if (
SDNode *E = lookupNode(ID,
DL, InsertToken)) {
11979 auto *
N = newSDNode<VPLoadFFSDNode>(
DL.getIROrder(),
DL.getDebugLoc(), VTs,
11981 createOperands(
N,
Ops);
11983 CSEMap.insert(
N, InsertToken);
11997 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
12002 if (
SDNode *E = lookupNode(ID, dl, InsertToken))
12007 createOperands(
N,
Ops);
12009 CSEMap.insert(
N, InsertToken);
12023 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
12028 if (
SDNode *E = lookupNode(ID, dl, InsertToken))
12033 createOperands(
N,
Ops);
12035 CSEMap.insert(
N, InsertToken);
12046 if (
Cond.isUndef())
12081 return !Val || Val->getAPIntValue().uge(
X.getScalarValueSizeInBits());
12087 if (
X.getValueType().getScalarType() == MVT::i1)
12100 bool HasNan = (XC && XC->
getValueAPF().isNaN()) ||
12102 bool HasInf = (XC && XC->
getValueAPF().isInfinity()) ||
12105 if (Flags.hasNoNaNs() && (HasNan ||
X.isUndef() ||
Y.isUndef()))
12108 if (Flags.hasNoInfs() && (HasInf ||
X.isUndef() ||
Y.isUndef()))
12131 if (Opcode ==
ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
12146 switch (
Ops.size()) {
12147 case 0:
return getNode(Opcode,
DL, VT);
12157 return getNode(Opcode,
DL, VT, NewOps);
12164 Flags = Inserter->getFlags();
12172 case 0:
return getNode(Opcode,
DL, VT);
12173 case 1:
return getNode(Opcode,
DL, VT,
Ops[0], Flags);
12180 for (
const auto &
Op :
Ops)
12182 "Operand is DELETED_NODE!");
12199 "LHS and RHS of condition must have same type!");
12201 "True and False arms of SelectCC must have same type!");
12203 "select_cc node must be of same type as true and false value!");
12207 "Expected select_cc with vector result to have the same sized "
12208 "comparison type!");
12213 "LHS/RHS of comparison should match types!");
12215 case ISD::VP_REDUCE_MUL:
12218 Opcode = ISD::VP_REDUCE_AND;
12220 case ISD::VP_REDUCE_ADD:
12223 Opcode = ISD::VP_REDUCE_XOR;
12225 case ISD::VP_REDUCE_SMAX:
12226 case ISD::VP_REDUCE_UMIN:
12230 Opcode = ISD::VP_REDUCE_AND;
12232 case ISD::VP_REDUCE_SMIN:
12233 case ISD::VP_REDUCE_UMAX:
12237 Opcode = ISD::VP_REDUCE_OR;
12245 if (VT != MVT::Glue) {
12249 if (
SDNode *E = lookupNode(ID,
DL, InsertToken)) {
12250 E->intersectFlagsWith(Flags);
12254 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
12255 createOperands(
N,
Ops);
12257 CSEMap.insert(
N, InsertToken);
12259 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
12260 createOperands(
N,
Ops);
12263 N->setFlags(Flags);
12274 Flags = Inserter->getFlags();
12288 Flags = Inserter->getFlags();
12298 for (
const auto &
Op :
Ops)
12300 "Operand is DELETED_NODE!");
12312 "Invalid add/sub overflow op!");
12314 Ops[0].getValueType() ==
Ops[1].getValueType() &&
12315 Ops[0].getValueType() == VTList.
VTs[0] &&
12316 "Binary operator types must match!");
12323 if (N2CV && N2CV->
isZero()) {
12354 "Invalid add/sub overflow op!");
12356 Ops[0].getValueType() ==
Ops[1].getValueType() &&
12357 Ops[0].getValueType() == VTList.
VTs[0] &&
12358 Ops[2].getValueType() == VTList.
VTs[1] &&
12359 "Binary operator types must match!");
12363 assert(VTList.
NumVTs == 2 &&
Ops.size() == 2 &&
"Invalid mul lo/hi op!");
12365 VTList.
VTs[0] ==
Ops[0].getValueType() &&
12366 VTList.
VTs[0] ==
Ops[1].getValueType() &&
12367 "Binary operator types must match!");
12371 assert(VTList.
NumVTs == 2 &&
Ops.size() == 1 &&
"Invalid ffrexp op!");
12373 VTList.
VTs[0] ==
Ops[0].getValueType() &&
"frexp type mismatch");
12381 DL, VTList.
VTs[1]);
12389 "Invalid STRICT_FP_EXTEND!");
12391 Ops[1].getValueType().isFloatingPoint() &&
"Invalid FP cast!");
12393 "STRICT_FP_EXTEND result type should be vector iff the operand "
12394 "type is vector!");
12397 Ops[1].getValueType().getVectorElementCount()) &&
12398 "Vector element count mismatch!");
12400 "Invalid fpext node, dst <= src!");
12403 assert(VTList.
NumVTs == 2 &&
Ops.size() == 3 &&
"Invalid STRICT_FP_ROUND!");
12405 "STRICT_FP_ROUND result type should be vector iff the operand "
12406 "type is vector!");
12409 Ops[1].getValueType().getVectorElementCount()) &&
12410 "Vector element count mismatch!");
12412 Ops[1].getValueType().isFloatingPoint() &&
12415 (
Ops[2]->getAsZExtVal() == 0 ||
Ops[2]->getAsZExtVal() == 1) &&
12416 "Invalid STRICT_FP_ROUND!");
12422 if (VTList.
VTs[VTList.
NumVTs-1] != MVT::Glue) {
12425 if (
SDNode *E = lookupNode(ID,
DL, InsertToken)) {
12426 E->intersectFlagsWith(Flags);
12430 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTList);
12431 createOperands(
N,
Ops);
12432 CSEMap.insert(
N, InsertToken);
12434 N = newSDNode<SDNode>(Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTList);
12435 createOperands(
N,
Ops);
12438 N->setFlags(Flags);
12490 EVT VTs[] = {VT1, VT2};
12495 EVT VTs[] = {VT1, VT2, VT3};
12500 EVT VTs[] = {VT1, VT2, VT3, VT4};
12505 auto It = VTLists.find(VTs);
12506 if (It == VTLists.end()) {
12507 EVT *Array = Allocator.Allocate<
EVT>(VTs.
size());
12509 It = VTLists.insert(
ArrayRef(Array, VTs.
size())).first;
12521 assert(
N->getNumOperands() == 1 &&
"Update with wrong number of operands");
12524 if (
Op ==
N->getOperand(0))
return N;
12528 if (
SDNode *Existing = FindModifiedNodeSlot(
N,
Op, InsertToken)) {
12535 if (!RemoveNodeFromCSEMaps(
N))
12539 N->OperandList[0].set(
Op);
12544 CSEMap.insert(
N, InsertToken);
12549 assert(
N->getNumOperands() == 2 &&
"Update with wrong number of operands");
12552 if (Op1 ==
N->getOperand(0) && Op2 ==
N->getOperand(1))
12557 if (
SDNode *Existing = FindModifiedNodeSlot(
N, Op1, Op2, InsertToken)) {
12564 if (!RemoveNodeFromCSEMaps(
N))
12568 if (
N->OperandList[0] != Op1)
12569 N->OperandList[0].set(Op1);
12570 if (
N->OperandList[1] != Op2)
12571 N->OperandList[1].set(Op2);
12576 CSEMap.insert(
N, InsertToken);
12596 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
12604 "Update with wrong number of operands");
12607 if (std::equal(
Ops.begin(),
Ops.end(),
N->op_begin()))
12612 if (
SDNode *Existing = FindModifiedNodeSlot(
N,
Ops, InsertToken)) {
12619 if (!RemoveNodeFromCSEMaps(
N))
12623 for (
unsigned i = 0; i !=
NumOps; ++i)
12624 if (
N->OperandList[i] !=
Ops[i])
12625 N->OperandList[i].set(
Ops[i]);
12630 CSEMap.insert(
N, InsertToken);
12647 if (NewMemRefs.
empty()) {
12653 if (NewMemRefs.
size() == 1) {
12654 N->MemRefs = NewMemRefs[0];
12660 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.
size());
12662 N->MemRefs = MemRefsBuffer;
12663 N->NumMemRefs =
static_cast<int>(NewMemRefs.
size());
12735 New->setNodeId(-1);
12755 unsigned Order = std::min(
N->getIROrder(), OLoc.
getIROrder());
12756 N->setIROrder(Order);
12780 if (VTs.
VTs[VTs.
NumVTs-1] != MVT::Glue) {
12783 if (
SDNode *ON = lookupNode(ID,
SDLoc(
N), InsertToken))
12784 return UpdateSDLocOnMergeSDNode(ON,
SDLoc(
N));
12787 if (!RemoveNodeFromCSEMaps(
N))
12792 N->ValueList = VTs.
VTs;
12802 if (Used->use_empty())
12803 DeadNodeSet.
insert(Used);
12808 MN->clearMemRefs();
12812 createOperands(
N,
Ops);
12816 if (!DeadNodeSet.
empty()) {
12818 for (
SDNode *
N : DeadNodeSet)
12819 if (
N->use_empty())
12825 CSEMap.insert(
N, InsertToken);
12830 unsigned OrigOpc =
Node->getOpcode();
12835#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12836 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
12837#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12838 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
12839#include "llvm/IR/ConstrainedOps.def"
12842 assert(
Node->getNumValues() == 2 &&
"Unexpected number of results!");
12850 for (
unsigned i = 1, e =
Node->getNumOperands(); i != e; ++i)
12851 Ops.push_back(
Node->getOperand(i));
12968 bool DoCSE = VTs.
VTs[VTs.
NumVTs-1] != MVT::Glue;
12974 if (
SDNode *E = lookupNode(ID,
DL, InsertToken)) {
12980 N = newSDNode<MachineSDNode>(~Opcode,
DL.getIROrder(),
DL.getDebugLoc(), VTs);
12981 createOperands(
N,
Ops);
12984 CSEMap.insert(
N, InsertToken);
12997 VT, Operand, SRIdxVal);
13007 VT, Operand, Subreg, SRIdxVal);
13015 bool AllowCommute) {
13018 Flags = Inserter->getFlags();
13025 bool AllowCommute) {
13026 if (VTList.
VTs[VTList.
NumVTs - 1] == MVT::Glue)
13030 SDNodeKey ID(Opcode, VTList, LookupOps);
13032 if (
SDNode *E = lookupNode(ID, InsertToken)) {
13033 E->intersectFlagsWith(Flags);
13042 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))
13051 if (VTList.
VTs[VTList.
NumVTs - 1] != MVT::Glue) {
13054 if (lookupNode(ID,
SDLoc(), InsertToken))
13064 SDNode *
N,
unsigned R,
bool IsIndirect,
13067 "Expected inlined-at fields to agree");
13068 return new (DbgInfo->getAlloc())
13070 {}, IsIndirect,
DL, O,
13080 "Expected inlined-at fields to agree");
13081 return new (DbgInfo->getAlloc())
13094 "Expected inlined-at fields to agree");
13106 "Expected inlined-at fields to agree");
13107 return new (DbgInfo->getAlloc())
13109 Dependencies, IsIndirect,
DL, O,
13118 "Expected inlined-at fields to agree");
13119 return new (DbgInfo->getAlloc())
13121 {}, IsIndirect,
DL, O,
13129 unsigned O,
bool IsVariadic) {
13131 "Expected inlined-at fields to agree");
13132 return new (DbgInfo->getAlloc())
13133 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
13134 DL, O, IsVariadic);
13138 unsigned OffsetInBits,
unsigned SizeInBits,
13139 bool InvalidateDbg) {
13142 assert(FromNode && ToNode &&
"Can't modify dbg values");
13147 if (From == To || FromNode == ToNode)
13159 if (Dbg->isInvalidated())
13167 auto NewLocOps = Dbg->copyLocationOps();
13169 NewLocOps.begin(), NewLocOps.end(),
13171 bool Match = Op == FromLocOp;
13181 auto *Expr = Dbg->getExpression();
13187 if (
auto FI = Expr->getFragmentInfo())
13188 if (OffsetInBits + SizeInBits > FI->SizeInBits)
13197 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
13200 Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),
13201 Dbg->getDebugLoc(), std::max(ToNode->
getIROrder(), Dbg->getOrder()),
13202 Dbg->isVariadic());
13205 if (InvalidateDbg) {
13207 Dbg->setIsInvalidated();
13208 Dbg->setIsEmitted();
13214 "Transferred DbgValues should depend on the new SDNode");
13220 if (!
N.getHasDebugValue())
13223 auto GetLocationOperand = [](
SDNode *
Node,
unsigned ResNo) {
13231 if (DV->isInvalidated())
13233 switch (
N.getOpcode()) {
13243 Offset =
N.getConstantOperandVal(1);
13246 if (!RHSConstant && DV->isIndirect())
13253 auto *DIExpr = DV->getExpression();
13254 auto NewLocOps = DV->copyLocationOps();
13256 size_t OrigLocOpsSize = NewLocOps.size();
13257 for (
size_t i = 0; i < OrigLocOpsSize; ++i) {
13262 NewLocOps[i].getSDNode() != &
N)
13273 const auto *TmpDIExpr =
13281 NewLocOps.push_back(RHS);
13290 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
13292 auto AdditionalDependencies = DV->getAdditionalDependencies();
13294 DV->getVariable(), DIExpr, NewLocOps, AdditionalDependencies,
13295 DV->isIndirect(), DV->getDebugLoc(), DV->getOrder(), IsVariadic);
13297 DV->setIsInvalidated();
13298 DV->setIsEmitted();
13300 N0.
getNode()->dumprFull(
this);
13301 dbgs() <<
" into " << *DIExpr <<
'\n');
13308 TypeSize ToSize =
N.getValueSizeInBits(0);
13312 auto NewLocOps = DV->copyLocationOps();
13314 for (
size_t i = 0; i < NewLocOps.size(); ++i) {
13316 NewLocOps[i].getSDNode() != &
N)
13328 DV->getAdditionalDependencies(), DV->isIndirect(),
13329 DV->getDebugLoc(), DV->getOrder(), DV->isVariadic());
13332 DV->setIsInvalidated();
13333 DV->setIsEmitted();
13335 dbgs() <<
" into " << *DbgExpression <<
'\n');
13342 assert((!Dbg->getSDNodes().empty() ||
13345 return Op.getKind() == SDDbgOperand::FRAMEIX;
13347 "Salvaged DbgValue should depend on a new SDNode");
13356 "Expected inlined-at fields to agree");
13357 return new (DbgInfo->getAlloc())
SDDbgLabel(Label,
DL, O);
13372 while (UI != UE &&
N == UI->
getUser())
13380 :
SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
13393 "Cannot replace with this method!");
13394 assert(From != To.
getNode() &&
"Cannot replace uses of with self");
13409 RAUWUpdateListener Listener(*
this, UI, UE);
13414 RemoveNodeFromCSEMaps(
User);
13429 AddModifiedNodeToCSEMaps(
User);
13445 for (
unsigned i = 0, e = From->
getNumValues(); i != e; ++i)
13448 "Cannot use this version of ReplaceAllUsesWith!");
13456 for (
unsigned i = 0, e = From->
getNumValues(); i != e; ++i)
13458 assert((i < To->getNumValues()) &&
"Invalid To location");
13467 RAUWUpdateListener Listener(*
this, UI, UE);
13472 RemoveNodeFromCSEMaps(
User);
13488 AddModifiedNodeToCSEMaps(
User);
13505 for (
unsigned i = 0, e = From->
getNumValues(); i != e; ++i) {
13515 RAUWUpdateListener Listener(*
this, UI, UE);
13520 RemoveNodeFromCSEMaps(
User);
13526 bool To_IsDivergent =
false;
13541 AddModifiedNodeToCSEMaps(
User);
13554 if (From == To)
return;
13570 RAUWUpdateListener Listener(*
this, UI, UE);
13573 bool UserRemovedFromCSEMaps =
false;
13590 if (!UserRemovedFromCSEMaps) {
13591 RemoveNodeFromCSEMaps(
User);
13592 UserRemovedFromCSEMaps =
true;
13602 if (!UserRemovedFromCSEMaps)
13607 AddModifiedNodeToCSEMaps(
User);
13626bool operator<(
const UseMemo &L,
const UseMemo &R) {
13627 return (intptr_t)L.User < (intptr_t)R.User;
13634 SmallVectorImpl<UseMemo> &
Uses;
13636 void NodeDeleted(SDNode *
N, SDNode *
E)
override {
13637 for (UseMemo &Memo :
Uses)
13638 if (Memo.User ==
N)
13639 Memo.User =
nullptr;
13643 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
13644 : SelectionDAG::DAGUpdateListener(d),
Uses(uses) {}
13651 switch (
Node->getOpcode()) {
13663 if (TLI->isSDNodeAlwaysUniform(
N)) {
13664 assert(!TLI->isSDNodeSourceOfDivergence(
N, FLI, UA) &&
13665 "Conflicting divergence information!");
13668 if (TLI->isSDNodeSourceOfDivergence(
N, FLI, UA))
13670 for (
const auto &
Op :
N->ops()) {
13671 EVT VT =
Op.getValueType();
13674 if (VT != MVT::Other &&
Op.getNode()->isDivergent() &&
13686 if (
N->SDNodeBits.IsDivergent != IsDivergent) {
13687 N->SDNodeBits.IsDivergent = IsDivergent;
13690 }
while (!Worklist.
empty());
13693void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
13695 Order.reserve(AllNodes.size());
13697 unsigned NOps =
N.getNumOperands();
13700 Order.push_back(&
N);
13702 for (
size_t I = 0;
I != Order.size(); ++
I) {
13704 for (
auto *U :
N->users()) {
13705 unsigned &UnsortedOps = Degree[U];
13706 if (0 == --UnsortedOps)
13707 Order.push_back(U);
13712#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
13713void SelectionDAG::VerifyDAGDivergence() {
13714 std::vector<SDNode *> TopoOrder;
13715 CreateTopologicalOrder(TopoOrder);
13716 for (
auto *
N : TopoOrder) {
13718 "Divergence bit inconsistency detected");
13741 for (
unsigned i = 0; i != Num; ++i) {
13742 unsigned FromResNo = From[i].
getResNo();
13745 if (
Use.getResNo() == FromResNo) {
13747 Uses.push_back(Memo);
13754 RAUOVWUpdateListener Listener(*
this,
Uses);
13756 for (
unsigned UseIndex = 0, UseIndexEnd =
Uses.size();
13757 UseIndex != UseIndexEnd; ) {
13763 if (
User ==
nullptr) {
13769 RemoveNodeFromCSEMaps(
User);
13776 unsigned i =
Uses[UseIndex].Index;
13781 }
while (UseIndex != UseIndexEnd &&
Uses[UseIndex].
User ==
User);
13785 AddModifiedNodeToCSEMaps(
User);
13793 unsigned DAGSize = 0;
13809 unsigned Degree =
N.getNumOperands();
13812 N.setNodeId(DAGSize++);
13814 if (Q != SortedPos)
13815 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
13816 assert(SortedPos != AllNodes.end() &&
"Overran node list");
13820 N.setNodeId(Degree);
13832 unsigned Degree =
P->getNodeId();
13833 assert(Degree != 0 &&
"Invalid node degree");
13837 P->setNodeId(DAGSize++);
13838 if (
P->getIterator() != SortedPos)
13839 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(
P));
13840 assert(SortedPos != AllNodes.end() &&
"Overran node list");
13844 P->setNodeId(Degree);
13847 if (
Node.getIterator() == SortedPos) {
13851 dbgs() <<
"Overran sorted position:\n";
13853 dbgs() <<
"Checking if this is due to cycles\n";
13860 assert(SortedPos == AllNodes.end() &&
13861 "Topological sort incomplete!");
13863 "First node in topological sort is not the entry token!");
13864 assert(AllNodes.front().getNodeId() == 0 &&
13865 "First node in topological sort has non-zero id!");
13866 assert(AllNodes.front().getNumOperands() == 0 &&
13867 "First node in topological sort has operands!");
13868 assert(AllNodes.back().getNodeId() == (
int)DAGSize-1 &&
13869 "Last node in topologic sort has unexpected id!");
13870 assert(AllNodes.back().use_empty() &&
13871 "Last node in topologic sort has users!");
13878 SortedNodes.
clear();
13885 unsigned NumOperands =
N.getNumOperands();
13886 if (NumOperands == 0)
13890 RemainingOperands[&
N] = NumOperands;
13895 for (
unsigned i = 0U; i < SortedNodes.
size(); ++i) {
13896 const SDNode *
N = SortedNodes[i];
13897 for (
const SDNode *U :
N->users()) {
13902 unsigned &NumRemOperands = RemainingOperands[U];
13903 assert(NumRemOperands &&
"Invalid number of remaining operands");
13905 if (!NumRemOperands)
13910 assert(SortedNodes.
size() == AllNodes.size() &&
"Node count mismatch");
13912 "First node in topological sort is not the entry token");
13913 assert(SortedNodes.
front()->getNumOperands() == 0 &&
13914 "First node in topological sort has operands");
13920 for (
SDNode *SD : DB->getSDNodes()) {
13923 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
13924 SD->setHasDebugValue(
true);
13926 DbgInfo->add(DB, isParameter);
13939 if (OldChain == NewMemOpChain || OldChain.
use_empty())
13940 return NewMemOpChain;
13943 OldChain, NewMemOpChain);
13946 return TokenFactor;
13965 if (OutFunction !=
nullptr)
13973 std::string ErrorStr;
13975 ErrorFormatter <<
"Undefined external symbol ";
13976 ErrorFormatter <<
'"' << Symbol <<
'"';
13986 return Const !=
nullptr && Const->isZero();
13995 return Const !=
nullptr && Const->isZero() && !Const->isNegative();
14000 return Const !=
nullptr && Const->isAllOnes();
14005 return Const !=
nullptr && Const->isOne();
14010 return Const !=
nullptr && Const->isMinSignedValue();
14014 SDValue V,
unsigned OperandNo,
14015 unsigned Depth)
const {
14022 unsigned OperandNo,
unsigned Depth)
const {
14025 if (V.getValueType().isInteger()) {
14027 if (
Known.isConstant()) {
14034 return Const.isZero();
14036 return Const.isOne();
14039 return Const.isAllOnes();
14041 return Const.isMinSignedValue();
14043 return Const.isMaxSignedValue();
14048 return OperandNo == 1 && Const.isZero();
14051 return OperandNo == 1 && Const.isOne();
14057 return ConstFP->isZero() &&
14058 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
14060 return OperandNo == 1 && ConstFP->isZero() &&
14061 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
14063 return ConstFP->isOne();
14065 return OperandNo == 1 && ConstFP->isOne();
14072 EVT VT = V.getValueType();
14080 return ConstFP->isExactlyValue(NeutralAF);
14085 const APFloat &VAPF = ConstFP->getValueAPF();
14087 if (Flags.hasNoInfs())
14103 while (V.getOpcode() ==
ISD::BITCAST && V.getOperand(0).hasOneUse())
14122 !DemandedElts[IndexC->getZExtValue()]) {
14141 unsigned NumBits = V.getScalarValueSizeInBits();
14144 return C && (
C->getAPIntValue().
countr_one() >= NumBits);
14148 bool AllowTruncation) {
14155 bool AllowTruncation) {
14162 EVT VecEltVT =
N->getValueType(0).getVectorElementType();
14164 EVT CVT = CN->getValueType(0);
14165 assert(CVT.
bitsGE(VecEltVT) &&
"Illegal splat_vector element extension");
14166 if (AllowTruncation || CVT == VecEltVT)
14173 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
14178 if (CN && (UndefElements.
none() || AllowUndefs)) {
14180 EVT NSVT =
N.getValueType().getScalarType();
14181 assert(CVT.
bitsGE(NSVT) &&
"Illegal build vector element extension");
14182 if (AllowTruncation || (CVT == NSVT))
14189 AllowUndefs, AllowTruncation);
14200 const APInt &DemandedElts,
14201 bool AllowUndefs) {
14208 BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
14210 if (CN && (UndefElements.
none() || AllowUndefs))
14225 return C &&
C->isZero();
14231 return C &&
C->isOne();
14236 return C &&
C->isOne();
14241 unsigned BitWidth =
N.getScalarValueSizeInBits();
14244 return C &&
C->getAPIntValue().countTrailingOnes() >=
BitWidth;
14250 return C &&
C->isZero();
14255 return C &&
C->isZero();
14266 bool IsVolatile =
false;
14267 bool IsNonTemporal =
false;
14268 bool IsDereferenceable =
true;
14269 bool IsInvariant =
true;
14271 IsVolatile |= MMO->isVolatile();
14272 IsNonTemporal |= MMO->isNonTemporal();
14273 IsDereferenceable &= MMO->isDereferenceable();
14274 IsInvariant &= MMO->isInvariant();
14294 std::vector<EVT> VTs;
14307const EVT *SDNode::getValueTypeList(
MVT VT) {
14308 static EVTArray SimpleVTArray;
14311 return &SimpleVTArray.VTs[VT.
SimpleTy];
14320 if (U.getResNo() ==
Value)
14358 return any_of(
N->op_values(),
14359 [
this](
SDValue Op) { return this == Op.getNode(); });
14373 unsigned Depth)
const {
14374 if (*
this == Dest)
return true;
14378 if (
Depth == 0)
return false;
14398 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
14404 if (Ld->isUnordered())
14405 return Ld->getChain().reachesChainWithoutSideEffects(Dest,
Depth-1);
14418 this->Flags &= Flags;
14424 bool AllowPartials) {
14439 unsigned CandidateBinOp =
Op.getOpcode();
14440 if (
Op.getValueType().isFloatingPoint()) {
14442 switch (CandidateBinOp) {
14444 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
14454 auto PartialReduction = [&](
SDValue Op,
unsigned NumSubElts) {
14455 if (!AllowPartials || !
Op)
14457 EVT OpVT =
Op.getValueType();
14460 if (TLI->getExtractSubvectorCost(SubVT, OpVT, 0) >
14480 unsigned Stages =
Log2_32(
Op.getValueType().getVectorNumElements());
14482 for (
unsigned i = 0; i < Stages; ++i) {
14483 unsigned MaskEnd = (1 << i);
14485 if (
Op.getOpcode() != CandidateBinOp)
14486 return PartialReduction(PrevOp, MaskEnd);
14502 return PartialReduction(PrevOp, MaskEnd);
14505 for (
int Index = 0; Index < (int)MaskEnd; ++Index)
14506 if (Shuffle->
getMaskElt(Index) != (
int)(MaskEnd + Index))
14507 return PartialReduction(PrevOp, MaskEnd);
14514 while (
Op.getOpcode() == CandidateBinOp) {
14515 unsigned NumElts =
Op.getValueType().getVectorNumElements();
14524 if (NumSrcElts != (2 * NumElts))
14539 EVT VT =
N->getValueType(0);
14546 assert(NE &&
"Nothing to unroll!");
14552 else if (NE > ResNE)
14555 if (
N->getNumValues() == 2) {
14558 EVT VT1 =
N->getValueType(1);
14562 for (i = 0; i != NE; ++i) {
14563 for (
unsigned j = 0, e =
N->getNumOperands(); j != e; ++j) {
14564 SDValue Operand =
N->getOperand(j);
14577 for (; i < ResNE; ++i) {
14591 assert(
N->getNumValues() == 1 &&
14592 "Can't unroll a vector with multiple results!");
14598 for (i= 0; i != NE; ++i) {
14599 for (
unsigned j = 0, e =
N->getNumOperands(); j != e; ++j) {
14600 SDValue Operand =
N->getOperand(j);
14612 switch (
N->getOpcode()) {
14620 EVT CondVT =
Cond.getValueType();
14622 if (CondVT != MVT::i1 &&
14623 TLI.getBooleanContents(
N->getOperand(1).getValueType()) !=
14624 TLI.getBooleanContents(CondVT)) {
14654 ASC->getDestAddressSpace(), ASC->getFlags()));
14660 for (; i < ResNE; ++i)
14671 unsigned Opcode =
N->getOpcode();
14675 "Expected an overflow opcode");
14677 EVT ResVT =
N->getValueType(0);
14678 EVT OvVT =
N->getValueType(1);
14687 else if (NE > ResNE)
14699 for (
unsigned i = 0; i < NE; ++i) {
14700 SDValue Res =
getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
14723 if (LS->isVolatile() ||
Base->isVolatile())
14726 if (!LS->isSimple())
14728 if (LS->isIndexed() ||
Base->isIndexed())
14730 if (LS->getChain() !=
Base->getChain())
14732 EVT VT = LS->getMemoryVT();
14741 if (!BaseLocDecomp.equalBaseIndex(LocDecomp, DAG,
Offset))
14769 int64_t GVOffset = 0;
14770 if (TLI->isGAPlusOffset(Ptr.
getNode(), GV, GVOffset)) {
14774 unsigned AlignBits =
Known.countMinTrailingZeros();
14782 int FrameIdx = INT_MIN;
14783 int64_t FrameOffset = 0;
14785 FrameIdx = FI->getIndex();
14793 if (FrameIdx != INT_MIN) {
14798 return std::nullopt;
14808 "Split node must be a scalar type");
14813 return std::make_pair(
Lo,
Hi);
14822 LoVT = HiVT = TLI->getTypeToTransformTo(*
getContext(), VT);
14826 return std::make_pair(LoVT, HiVT);
14834 bool *HiIsEmpty)
const {
14844 "Mixing fixed width and scalable vectors when enveloping a type");
14849 *HiIsEmpty =
false;
14857 return std::make_pair(LoVT, HiVT);
14862std::pair<SDValue, SDValue>
14867 "Splitting vector with an invalid mixture of fixed and scalable "
14870 N.getValueType().getVectorMinNumElements() &&
14871 "More vector elements requested than available!");
14879 return std::make_pair(
Lo,
Hi);
14886 EVT VT =
N.getValueType();
14888 "Expecting the mask to be an evenly-sized vector");
14893 return std::make_pair(
Lo,
Hi);
14898 EVT VT =
N.getValueType();
14907 unsigned Start,
unsigned Count,
14909 EVT VT =
Op.getValueType();
14912 if (EltVT ==
EVT())
14915 for (
unsigned i = Start, e = Start +
Count; i != e; ++i) {
14927 return Val.MachineCPVal->getType();
14928 return Val.ConstVal->getType();
14932 unsigned &SplatBitSize,
14933 bool &HasAnyUndefs,
14934 unsigned MinSplatBits,
14935 bool IsBigEndian)
const {
14939 if (MinSplatBits > VecWidth)
14944 SplatValue =
APInt(VecWidth, 0);
14945 SplatUndef =
APInt(VecWidth, 0);
14952 assert(
NumOps > 0 &&
"isConstantSplat has 0-size build vector");
14955 for (
unsigned j = 0; j <
NumOps; ++j) {
14956 unsigned i = IsBigEndian ?
NumOps - 1 - j : j;
14958 unsigned BitPos = j * EltWidth;
14961 SplatUndef.
setBits(BitPos, BitPos + EltWidth);
14963 SplatValue.
insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
14965 SplatValue.
insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
14972 HasAnyUndefs = (SplatUndef != 0);
14975 while (VecWidth > 8) {
14980 unsigned HalfSize = VecWidth / 2;
14987 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
14988 MinSplatBits > HalfSize)
14991 SplatValue = HighValue | LowValue;
14992 SplatUndef = HighUndef & LowUndef;
14994 VecWidth = HalfSize;
15003 SplatBitSize = VecWidth;
15010 if (UndefElements) {
15011 UndefElements->
clear();
15018 for (
unsigned i = 0; i !=
NumOps; ++i) {
15019 if (!DemandedElts[i])
15022 if (
Op.isUndef()) {
15024 (*UndefElements)[i] =
true;
15025 }
else if (!Splatted) {
15027 }
else if (Splatted !=
Op) {
15033 unsigned FirstDemandedIdx = DemandedElts.
countr_zero();
15035 "Can only have a splat without a constant for all undefs.");
15052 if (UndefElements) {
15053 UndefElements->
clear();
15064 (*UndefElements)[
I] =
true;
15067 for (
unsigned SeqLen = 1; SeqLen <
NumOps; SeqLen *= 2) {
15068 Sequence.append(SeqLen,
SDValue());
15069 for (
unsigned I = 0;
I !=
NumOps; ++
I) {
15070 if (!DemandedElts[
I])
15072 SDValue &SeqOp = Sequence[
I % SeqLen];
15074 if (
Op.isUndef()) {
15079 if (SeqOp && !SeqOp.
isUndef() && SeqOp !=
Op) {
15085 if (!Sequence.empty())
15089 assert(Sequence.empty() &&
"Failed to empty non-repeating sequence pattern");
15130 const APFloat &APF = CN->getValueAPF();
15136 return IntVal.exactLogBase2();
15142 bool IsLittleEndian,
unsigned DstEltSizeInBits,
15150 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
15151 "Invalid bitcast scale");
15155 if ((SrcEltSizeInBits % DstEltSizeInBits) != 0 &&
15156 (DstEltSizeInBits % SrcEltSizeInBits) != 0)
15162 BitVector SrcUndeElements(NumSrcOps,
false);
15164 for (
unsigned I = 0;
I != NumSrcOps; ++
I) {
15166 if (
Op.isUndef()) {
15167 SrcUndeElements.
set(
I);
15172 assert((CInt || CFP) &&
"Unknown constant");
15173 SrcBitElements[
I] = CInt ? CInt->getAPIntValue().trunc(SrcEltSizeInBits)
15174 : CFP->getValueAPF().bitcastToAPInt();
15178 recastRawBits(IsLittleEndian, DstEltSizeInBits, RawBitElements,
15179 SrcBitElements, UndefElements, SrcUndeElements);
15184 unsigned DstEltSizeInBits,
15189 unsigned NumSrcOps = SrcBitElements.
size();
15190 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
15191 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
15192 "Invalid bitcast scale");
15193 assert(((SrcEltSizeInBits % DstEltSizeInBits) == 0 ||
15194 (DstEltSizeInBits % SrcEltSizeInBits) == 0) &&
15195 "Unsupported bitcast scale");
15196 assert(NumSrcOps == SrcUndefElements.
size() &&
"Vector size mismatch");
15198 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
15199 DstUndefElements.
clear();
15200 DstUndefElements.
resize(NumDstOps,
false);
15204 if (SrcEltSizeInBits <= DstEltSizeInBits) {
15205 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
15206 for (
unsigned I = 0;
I != NumDstOps; ++
I) {
15207 DstUndefElements.
set(
I);
15208 APInt &DstBits = DstBitElements[
I];
15209 for (
unsigned J = 0; J != Scale; ++J) {
15210 unsigned Idx = (
I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
15211 if (SrcUndefElements[Idx])
15213 DstUndefElements.
reset(
I);
15214 const APInt &SrcBits = SrcBitElements[Idx];
15216 "Illegal constant bitwidths");
15217 DstBits.
insertBits(SrcBits, J * SrcEltSizeInBits);
15224 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
15225 for (
unsigned I = 0;
I != NumSrcOps; ++
I) {
15226 if (SrcUndefElements[
I]) {
15227 DstUndefElements.
set(
I * Scale, (
I + 1) * Scale);
15230 const APInt &SrcBits = SrcBitElements[
I];
15231 for (
unsigned J = 0; J != Scale; ++J) {
15232 unsigned Idx = (
I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
15233 APInt &DstBits = DstBitElements[Idx];
15234 DstBits = SrcBits.
extractBits(DstEltSizeInBits, J * DstEltSizeInBits);
15241 unsigned Opc =
Op.getOpcode();
15248std::optional<std::pair<APInt, APInt>>
15252 return std::nullopt;
15255 APInt Start, Stride;
15256 int FirstIdx = -1, SecondIdx = -1;
15260 for (
unsigned I = 0;
I <
NumOps; ++
I) {
15265 return std::nullopt;
15268 if (FirstIdx < 0) {
15271 }
else if (SecondIdx < 0) {
15277 unsigned IdxDiff =
I - FirstIdx;
15278 APInt ValDiff = Val - Start;
15283 return std::nullopt;
15284 IdxDiff >>= CommonPow2Bits;
15292 return std::nullopt;
15295 Start -= Stride * FirstIdx;
15298 if (Val != Start + Stride *
I)
15299 return std::nullopt;
15305 return std::nullopt;
15307 return std::make_pair(Start, Stride);
15313 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
15323 for (
int Idx = Mask[i]; i != e; ++i)
15324 if (Mask[i] >= 0 && Mask[i] != Idx)
15332 SDValue N,
bool AllowOpaques)
const {
15336 return AllowOpaques || !
C->isOpaque();
15345 TLI->isOffsetFoldingLegal(GA))
15373 return std::nullopt;
15375 EVT VT =
N.getValueType();
15377 switch (TLI->getBooleanContents(VT)) {
15383 return std::nullopt;
15389 return std::nullopt;
15397 assert(!
Node->OperandList &&
"Node already has operands");
15399 "too many operands to fit into SDNode");
15400 SDUse *
Ops = OperandRecycler.allocate(
15403 bool IsDivergent =
false;
15404 for (
unsigned I = 0;
I != Vals.
size(); ++
I) {
15406 Ops[
I].setInitial(Vals[
I]);
15407 EVT VT =
Ops[
I].getValueType();
15410 if (VT != MVT::Other &&
15413 IsDivergent =
true;
15418 if (!TLI->isSDNodeAlwaysUniform(Node)) {
15419 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
15420 Node->SDNodeBits.IsDivergent = IsDivergent;
15428 while (Vals.
size() > Limit) {
15429 unsigned SliceIdx = Vals.
size() - Limit;
15501 "Unexpected opcode");
15524 RTLIB::LibcallImpl LibcallImpl =
15525 Libcalls->getLibcallImpl(
static_cast<RTLIB::Libcall
>(LibFunc));
15526 if (LibcallImpl == RTLIB::Unsupported) {
15528 Node->getOperationName(
this));
15539 .
setLibCallee(Libcalls->getLibcallImplCallingConv(LibcallImpl),
15541 return TLI->LowerCallTo(CLI).second;
15545 assert(From && To &&
"Invalid SDNode; empty source SDValue?");
15546 auto I = SDEI.find(From);
15547 if (
I == SDEI.end())
15552 NodeExtraInfo NEI =
I->second;
15561 SDEI[To] = std::move(NEI);
15578 auto VisitFrom = [&](
auto &&Self,
const SDNode *
N,
int MaxDepth) {
15579 if (MaxDepth == 0) {
15585 if (!FromReach.
insert(
N).second)
15588 Self(Self,
Op.getNode(), MaxDepth - 1);
15593 auto DeepCopyTo = [&](
auto &&Self,
const SDNode *
N) {
15596 if (!Visited.
insert(
N).second)
15601 if (
N == To &&
Op.getNode() == EntrySDN) {
15606 if (!Self(Self,
Op.getNode()))
15610 SDEI[
N] = std::move(NEI);
15620 for (
int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
15621 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.
clear()) {
15626 for (
const SDNode *
N : StartFrom)
15627 VisitFrom(VisitFrom,
N, MaxDepth - PrevDepth);
15631 LLVM_DEBUG(
dbgs() << __func__ <<
": MaxDepth=" << MaxDepth <<
" too low\n");
15639 errs() <<
"warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
15640 assert(
false &&
"From subgraph too complex - increase max. MaxDepth?");
15642 SDEI[To] = std::move(NEI);
15659 APInt MaxNElts = MinNElts.
umul_ov(MaxVScale, Overflow);
15669 "Element count mismatch!");
15693 if (!Visited.
insert(
N).second) {
15694 errs() <<
"Detected cycle in SelectionDAG\n";
15695 dbgs() <<
"Offending node:\n";
15696 N->dumprFull(DAG);
dbgs() <<
"\n";
15712 bool check = force;
15713#ifdef EXPENSIVE_CHECKS
15717 assert(
N &&
"Checking nonexistent SDNode");
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
This file implements the BitVector class.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
#define __asan_unpoison_memory_region(p, size)
#define LLVM_LIKELY(EXPR)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseSet and SmallDenseSet classes.
This file contains constants used for implementing Dwarf debug support.
This file defines a hash set that can be used to remove duplication of nodes in a graph.
static MaybeAlign getAlign(Value *Ptr)
std::pair< Instruction::BinaryOps, Value * > OffsetOp
Find all possible pairs (BinOp, RHS) that BinOp V, RHS can be simplified.
static constexpr Value * getValue(Ty &ValueOrUse)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
static bool isConstantSplatVector(SDValue N, APInt &SplatValue, unsigned MinSizeInBits)
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
Register const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
PowerPC Reduce CR logical Operation
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
Remove Loads Into Fake Uses
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
Contains matchers for matching SelectionDAG nodes and values.
static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow)
static bool shouldLowerMemFuncForSize(const MachineFunction &MF, SelectionDAG &DAG)
static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL, EVT VT, Ty Quantity)
static std::pair< SDValue, SDValue > getRuntimeCallSDValueHelper(SDValue Chain, const SDLoc &dl, TargetLowering::ArgListTy &&Args, const CallInst *CI, RTLIB::Libcall Call, SelectionDAG *DAG, const TargetLowering *TLI)
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo)
Lower the call to 'memset' intrinsic function into a series of store operations.
static std::optional< APInt > FoldValueWithUndef(unsigned Opcode, const APInt &C1, bool IsUndef1, const APInt &C2, bool IsUndef2)
static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step, SelectionDAG &DAG)
static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, const TargetLowering &TLI, const ConstantDataArraySlice &Slice)
getMemsetStringVal - Similar to getMemsetValue.
static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B)
static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N)
Serialize a node the way SDNodes were uniqued before SDNodeKey, to cross-check the typed comparison a...
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef< int > M)
Swaps the values of N1 and N2.
static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice)
Returns true if memcpy source is constant data.
static TargetLoweringBase::ArgListEntry getAtomicMemOpSizeArg(SelectionDAG *DAG, const SDLoc &dl, SDValue Size)
static bool hasSpecialNszSemantics(unsigned Opcode)
Return true if the standard nsz semantics may not apply for this opcode.
static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike)
static bool doNotCSE(SDNode *N)
doNotCSE - Return true if CSE should not be performed for this node.
static bool areNonVolatileConsecutiveLoadsOrStores(LSBaseSDNode *LS, LSBaseSDNode *Base, TypeSize Bytes, int Dist, const SelectionDAG &DAG)
static APInt getIntegerIdentity(unsigned Opcode, unsigned BitWidth)
static SDValue getMemIntrinsicSize(SelectionDAG &DAG, const SDLoc &dl, SDValue Size, unsigned DstAS, unsigned SrcAS)
The length of a memory intrinsic (e.g.
static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, const DataLayout &DL, unsigned AS)
static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
Try to simplify vector concatenation to an input value, undef, or build vector.
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, SelectionDAG &DAG, SDValue Ptr, int64_t Offset=0)
InferPointerInfo - If the specified ptr/offset is a frame index, infer a MachinePointerInfo record fr...
static bool isInTailCallPositionWrapper(const CallInst *CI, const SelectionDAG *SelDAG, bool AllowReturnsFirstArg)
static bool keyMatches(const SDNodeKey &Key, const SDNode &N)
static bool gluePropagatesDivergence(const SDNode *Node)
Return true if a glue output should propagate divergence information.
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N, unsigned Opc)
If this is an SDNode with special info, add this info to the NodeID data.
static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G)
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs)
makeVTList - Return an instance of the SDVTList struct initialized with the specified members.
static void checkForCyclesHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallPtrSetImpl< const SDNode * > &Checked, const llvm::SelectionDAG *DAG)
static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SmallVector< SDValue, 32 > &OutChains, unsigned From, unsigned To, SmallVector< SDValue, 16 > &OutLoadChains, SmallVector< SDValue, 16 > &OutStoreChains)
static int isSignedOp(ISD::CondCode Opcode)
For an integer comparison, return 1 if the comparison is a signed operation and 2 if the result is an...
static std::optional< APInt > FoldValue(unsigned Opcode, const APInt &C1, const APInt &C2)
static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
static APInt getDemandAllEltsMask(SDValue V)
Construct a DemandedElts mask which demands all elements of V.
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, BatchAAResults *BatchAA, const MDNode *DstMemCacheHint, const MDNode *SrcMemCacheHint)
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo)
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
This file describes how to lower LLVM code to machine code.
static void removeOperands(MachineInstr &MI, unsigned i)
static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR)
Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
static int Lookup(ArrayRef< TableEntry > Table, unsigned Opcode)
static unsigned getSize(unsigned Kind)
static const fltSemantics & IEEEsingle()
cmpResult
IEEE-754R 5.11: Floating Point Comparison Relations.
static constexpr roundingMode rmTowardZero
static const fltSemantics & BFloat()
static const fltSemantics & IEEEquad()
static const fltSemantics & IEEEdouble()
static constexpr roundingMode rmTowardNegative
static constexpr roundingMode rmNearestTiesToEven
static constexpr roundingMode rmTowardPositive
static const fltSemantics & IEEEhalf()
opStatus
IEEE-754R 7: Default exception handling.
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
LLVM_READONLY bool isOne() const
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
LLVM_ABI APInt umul_ov(const APInt &RHS, bool &Overflow) const
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
bool isMinSignedValue() const
Determine if this is the smallest signed value.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
LLVM_ABI APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
APInt abs() const
Get the absolute value.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
LLVM_ABI APInt reverseBits() const
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
bool sle(const APInt &RHS) const
Signed less or equal comparison.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned getNumSignBits() const
Computes the number of leading bits of this APInt that are equal to its sign bit.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static LLVM_ABI APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
LLVM_ABI APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
LLVM_ABI void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
unsigned logBase2() const
LLVM_ABI APInt multiplicativeInverse() const
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
APInt shl(unsigned shiftAmt) const
Left-shift function.
LLVM_ABI APInt byteSwap() const
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
void clearBits(unsigned LoBit, unsigned HiBit)
Clear the bits from LoBit (inclusive) to HiBit (exclusive) to 0.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
LLVM_ABI APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
bool isOne() const
Determine if this is a value of 1.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
unsigned getSrcAddressSpace() const
unsigned getDestAddressSpace() const
static Capacity get(size_t N)
Get the capacity of an array that can hold at least N elements.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
static LLVM_ABI BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, and fixed/scalable offsets.
LLVM Basic Block Representation.
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
BitVector & reset()
Reset all bits in the bitvector.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
void clear()
Removes all bits from the bitvector.
BitVector & set()
Set all bits in the bitvector.
bool none() const
Returns true if none of the bits are set.
size_type size() const
Returns the number of bits in this bitvector.
int64_t getOffset() const
unsigned getTargetFlags() const
const BlockAddress * getBlockAddress() const
The address of a basic block.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
A "pseudo-class" with methods for operating on BUILD_VECTORs.
LLVM_ABI bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &RawBitElements, BitVector &UndefElements) const
Extract the raw bit data from a build vector of Undef, Constant or ConstantFP node elements.
static LLVM_ABI void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &DstBitElements, ArrayRef< APInt > SrcBitElements, BitVector &DstUndefElements, const BitVector &SrcUndefElements)
Recast bit data SrcBitElements to DstEltSizeInBits wide elements.
LLVM_ABI bool getRepeatedSequence(const APInt &DemandedElts, SmallVectorImpl< SDValue > &Sequence, BitVector *UndefElements=nullptr) const
Find the shortest repeating sequence of values in the build vector.
LLVM_ABI ConstantFPSDNode * getConstantFPSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant FP or null if this is not a constant FP splat.
LLVM_ABI SDValue getSplatValue(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted value or a null value if this is not a splat.
LLVM_ABI bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
LLVM_ABI ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
LLVM_ABI int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
LLVM_ABI std::optional< std::pair< APInt, APInt > > isArithmeticSequence() const
If this BuildVector is constant and represents an arithmetic sequence "<a, a+n, a+2n,...
LLVM_ABI bool isConstant() const
This class represents a function call, abstracting a target machine's calling convention.
static LLVM_ABI bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
ConstantFP - Floating Point Values [float, double].
const APFloat & getValue() const
This is the shared class of boolean and integer constants.
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
const APInt & getValue() const
Return the constant as an APInt value reference.
MachineConstantPoolValue * getMachineCPVal() const
bool isMachineConstantPoolEntry() const
const Constant * getConstVal() const
LLVM_ABI Type * getType() const
unsigned getTargetFlags() const
This class represents a range of values.
PreferredRangeType
If represented precisely, the result of some range operations may consist of multiple disjoint ranges...
const APInt * getSingleElement() const
If this set contains a single element, return it, otherwise return null.
static LLVM_ABI ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
LLVM_ABI OverflowResult unsignedSubMayOverflow(const ConstantRange &Other) const
Return whether unsigned sub of the two ranges always/never overflows.
LLVM_ABI OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const
Return whether unsigned add of the two ranges always/never overflows.
LLVM_ABI KnownBits toKnownBits() const
Return known bits for values in this range.
LLVM_ABI ConstantRange zeroExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
LLVM_ABI APInt getSignedMin() const
Return the smallest signed value contained in the ConstantRange.
LLVM_ABI OverflowResult unsignedMulMayOverflow(const ConstantRange &Other) const
Return whether unsigned mul of the two ranges always/never overflows.
LLVM_ABI ConstantRange signExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
LLVM_ABI ConstantRange multiply(const ConstantRange &Other, unsigned NoWrapKind=0) const
Return a new range representing the possible values resulting from a multiplication of a value in thi...
LLVM_ABI bool contains(const APInt &Val) const
Return true if the specified value is in the set.
LLVM_ABI APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
LLVM_ABI ConstantRange intersectWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the intersection of this range with another range.
LLVM_ABI APInt getSignedMax() const
Return the largest signed value contained in the ConstantRange.
OverflowResult
Represents whether an operation on the given constant range is known to always or never overflow.
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
uint32_t getBitWidth() const
Get the bit width of this ConstantRange.
LLVM_ABI OverflowResult signedSubMayOverflow(const ConstantRange &Other) const
Return whether signed sub of the two ranges always/never overflows.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
LLVM_ABI Constant * getSplatValue(bool AllowPoison=false) const
If all elements of the vector constant have the same value, return that value.
LLVM_ABI Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
static LLVM_ABI ExtOps getExtOps(unsigned FromSize, unsigned ToSize, bool Signed)
Returns the ops for a zero- or sign-extension in a DIExpression.
static LLVM_ABI void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static LLVM_ABI DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static LLVM_ABI const DIExpression * convertToVariadicExpression(const DIExpression *Expr)
If Expr is a non-variadic expression (i.e.
static LLVM_ABI std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
Base class for variables.
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
LLVM_ABI IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
LLVM_ABI Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
LLVM_ABI unsigned getPointerTypeSizeInBits(Type *) const
The pointer representation size in bits for this type.
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Implements a dense probed hash-table based set.
static constexpr ElementCount getFixed(ScalarTy MinVal)
constexpr bool isScalar() const
Exactly one element.
const char * getSymbol() const
unsigned getTargetFlags() const
Insertion token: a failed lookup fills it in, the matching insert consumes it.
This class is used to gather all the unique data bits of a node.
Data structure describing the variable locations in a function.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
AttributeList getAttributes() const
Return the attribute list for this Function.
int64_t getOffset() const
LLVM_ABI unsigned getAddressSpace() const
unsigned getTargetFlags() const
const GlobalValue * getGlobal() const
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
PointerType * getType() const
Global values are always pointers.
This class is used to form a handle around another node that is persistent and is updated across invo...
const SDValue & getValue() const
static LLVM_ABI bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
KnownFPClass intersectWith(const KnownFPClass &RHS) const
static KnownFPClass getEmpty()
static LLVM_ABI KnownFPClass bitcast(const fltSemantics &FltSemantics, const KnownBits &Bits)
Report known values for a bitcast into a float with provided semantics.
This is an important class for using LLVM in a threaded context.
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
Base class for LoadSDNode and StoreSDNode.
Tracks which library functions to use for a particular subtarget or function.
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Return the lowering's selection of implementation call for Call.
This SDNode is used for LIFETIME_START/LIFETIME_END values.
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
const MDOperand & getOperand(unsigned I) const
static LLVM_ABI MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
static MVT getIntegerVT(unsigned BitWidth)
Abstract base class for all machine specific constantpool value subclasses.
virtual void addSelectionDAGCSEId(FoldingSetNodeID &ID)=0
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
size_t getNumMemOperands() const
Return the number of memory operands.
LLVM_ABI MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt, PointerUnion< MachineMemOperand *, MachineMemOperand ** > memrefs)
Constructor that supports single or multiple MMOs.
PointerUnion< MachineMemOperand *, MachineMemOperand ** > MemRefs
Memory reference information.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
ArrayRef< MachineMemOperand * > memoperands() const
Return the memory operands for this node.
unsigned getRawSubclassData() const
Return the SubclassData value, without HasDebugValue.
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
A Module instance is used to store all the information related to an LLVM module.
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Class to represent pointers.
static PointerType * getUnqual(LLVMContext &C)
This constructs an opaque pointer to an object in the default address space (address space zero).
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
unsigned getAddressSpace() const
Return the address space of the Pointer type.
A discriminated union of two or more pointer types, with the discriminator in the low bits of the poi...
bool isNull() const
Test if the pointer held in the union is null, regardless of which type it is.
Analysis providing profile information.
void Deallocate(SubClass *E)
Deallocate - Release storage for the pointed-to object.
Wrapper class representing virtual and physical registers.
Keeps track of dbg_value information through SDISel.
LLVM_ABI void add(SDDbgValue *V, bool isParameter)
LLVM_ABI void erase(const SDNode *Node)
Invalidate all DbgValues attached to the node and remove it from the Node-to-DbgValues map.
Holds the information from a dbg_label node through SDISel.
Holds the information for a single machine location through SDISel; either an SDNode,...
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(Register VReg)
static SDDbgOperand fromConst(const Value *Const)
@ SDNODE
Value is the result of an expression.
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
unsigned getIROrder() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
LLVM_ABI void dumprFull(const SelectionDAG *G=nullptr) const
printrFull to dbgs().
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
LLVM_ABI bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
static constexpr size_t getMaxNumOperands()
Return the maximum number of operands that a SDNode can hold.
iterator_range< use_iterator > uses()
MemSDNodeBitfields MemSDNodeBits
bool getHasDebugValue() const
SDNodeFlags getFlags() const
void setNodeId(int Id)
Set unique node id.
LLVM_ABI void intersectFlagsWith(const SDNodeFlags Flags)
Clear any flags in this node that aren't also set in Flags.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
static LLVM_ABI bool areOnlyUsersOf(ArrayRef< const SDNode * > Nodes, const SDNode *N)
Return true if all the users of N are contained in Nodes.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
LLVM_ABI bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
std::optional< APInt > bitcastToAPInt() const
LLVM_ABI bool hasPredecessor(const SDNode *N) const
Return true if N is a predecessor of this node.
LLVM_ABI bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Returns true if the node type is UNDEF or POISON.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
LLVM_ABI void DropOperands()
Release the operands and set this node to have zero operands.
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
Create an SDNode.
Represents a use of a SDNode.
SDNode * getUser()
This returns the SDNode that contains this Use.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
LLVM_ABI bool isOperandOf(const SDNode *N) const
Return true if the referenced return value is an operand of N.
LLVM_ABI bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
virtual void verifyTargetNode(const SelectionDAG &DAG, const SDNode *N) const
Checks that the given target-specific node is valid. Aborts if it is not.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC)
LLVM_ABI Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
LLVM_ABI SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op)
Return the specified value casted to the target's desired shift amount type.
LLVM_ABI std::pair< SDValue, SDValue > getMemccpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue C, SDValue Size, const CallInst *CI)
Lower a memccpy operation into a target library call and return the resulting chain and call result a...
LLVM_ABI SDValue emitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, const SDLoc &DL, SDValue Chain)
Emit a store/load combination to the stack.
LLVM_ABI bool isKnownNeverLogicalZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Test whether the given floating point SDValue (or all elements of it, if it is a vector) is known to ...
LLVM_ABI SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsExpanding=false)
SDValue getExtractVectorElt(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Extract element at Idx from Vec.
LLVM_ABI SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
LLVM_ABI SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
LLVM_ABI OverflowKind computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const
Determine if the result of the unsigned sub of 2 nodes can overflow.
LLVM_ABI unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
LLVM_ABI std::pair< SDValue, SDValue > getStrlen(SDValue Chain, const SDLoc &dl, SDValue Src, const CallInst *CI)
Lower a strlen operation into a target library call and return the resulting chain and call result as...
LLVM_ABI SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
LLVM_ABI unsigned getHasPredecessorMaxSteps() const
LLVM_ABI SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl, SDNodeFlags Flags={})
Constant fold a setcc to true or false.
bool isKnownNeverSNaN(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
LLVM_ABI SDValue makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, SDValue InChain, SDNode *Node)
Helper used to make a call to a library function that has one argument of pointer type.
LLVM_ABI std::optional< bool > isBoolConstant(SDValue N) const
Check if a value \op N is a constant using the target's BooleanContent for its type.
LLVM_ABI SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
LLVM_ABI ConstantRange computeConstantRange(SDValue Op, bool ForSigned, unsigned Depth=0) const
Determine the possible constant range of an integer or vector of integers.
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI void updateDivergence(SDNode *N)
LLVM_ABI SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
LLVM_ABI SDValue getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT MemVT, EVT VT, SDValue Chain, SDValue Ptr, MachineMemOperand *MMO)
LLVM_ABI SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags, bool AllowCommute=false)
Get the specified node if it's already available, or else return NULL.
LLVM_ABI SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
LLVM_ABI SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
LLVM_ABI const BasicBlock * getBasicBlock() const
LLVM_ABI SelectionDAG(const TargetMachine &TM, CodeGenOptLevel)
LLVM_ABI SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
LLVM_ABI SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
LLVM_ABI SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
LLVM_ABI bool isConstantIntBuildVectorOrConstantInt(SDValue N, bool AllowOpaques=true) const
Test whether the given value is a constant int or similar node.
LLVM_ABI void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
LLVM_ABI SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
LLVM_ABI SDValue getSymbolFunctionGlobalAddress(SDValue Op, Function **TargetFunction=nullptr)
Return a GlobalAddress of the function from the current module with name matching the given ExternalS...
LLVM_ABI std::optional< unsigned > getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
LLVM_ABI bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, TypeSize Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
OverflowKind
Used to represent the possible overflow behavior of an operation.
LLVM_ABI bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
LLVM_ABI SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS, const SDNodeFlags Flags=SDNodeFlags())
Return an AddrSpaceCastSDNode.
SDValue getExtractSubvector(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Return the VT typed sub-vector of Vec at Idx.
LLVM_ABI bool cannotBeOrderedNegativeFP(SDValue Op) const
Test whether the given float value is known to be positive.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI bool calculateDivergence(SDNode *N)
LLVM_ABI std::pair< SDValue, SDValue > getStrcmp(SDValue Chain, const SDLoc &dl, SDValue S0, SDValue S1, const CallInst *CI)
Lower a strcmp operation into a target library call and return the resulting chain and call result as...
LLVM_ABI SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
LLVM_ABI SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
LLVM_ABI SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
LLVM_ABI bool canIgnoreSignBitOfZero(const SDUse &Use) const
Check if the user of Use produces the same result regardless of the sign of a zero in the used float ...
LLVM_ABI bool SignBitIsZeroFP(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero, for a floating-point value.
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getInsertSubvector(const SDLoc &DL, SDValue Vec, SDValue SubVec, unsigned Idx)
Insert SubVec at the Idx element of Vec.
LLVM_ABI SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false, SDNodeFlags Flags={})
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
LLVM_ABI Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
LLVM_ABI bool shouldOptForSize() const
LLVM_ABI bool hasSwiftErrorArg() const
LLVM_ABI SDValue buildVectorFromUnrolledParts(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Scalars)
Returns a vector constructed from the scalar values in order.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
LLVM_ABI SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
const TargetLowering & getTargetLoweringInfo() const
LLVM_ABI bool isEqualTo(SDValue A, SDValue B) const
Test whether two SDValues are known to compare equal.
static constexpr unsigned MaxRecursionDepth
LLVM_ABI unsigned getMaxRuntimeNumElements(EVT VT) const
Returns the maximum runtime number of elements in VT if known, or 0 otherwise.
LLVM_ABI SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
bool isGuaranteedNotToBePoison(SDValue Op, unsigned Depth=0) const
Return true if this function can prove that Op is never poison.
LLVM_ABI SDValue getIdentityElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) identity element for the given opcode, if it exists.
LLVM_ABI SDValue expandVACopy(SDNode *Node)
Expand the specified ISD::VACOPY node as the Legalize pass would.
LLVM_ABI SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
LLVM_ABI APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
For each demanded element of a vector, see if it is known to be zero.
LLVM_ABI void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
LLVM_ABI std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
LLVM_ABI void salvageDebugInfo(SDNode &N)
To be invoked on an SDNode that is slated to be erased.
LLVM_ABI SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
LLVM_ABI std::pair< SDValue, SDValue > UnrollVectorOverflowOp(SDNode *N, unsigned ResNE=0)
Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
LLVM_ABI SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
LLVM_ABI bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
LLVM_ABI void DeleteNode(SDNode *N)
Remove the specified node from the system.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
LLVM_ABI SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
LLVM_ABI std::pair< SDValue, SDValue > getStrcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, const CallInst *CI)
Lower a strcpy operation into a target library call and return the resulting chain and call result as...
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
LLVM_ABI std::optional< unsigned > getValidShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has a uniform shift amount that is less than the element bit-width of the shi...
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
LLVM_ABI SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
LLVM_ABI bool isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getPartialReduceMLS(unsigned Opc, const SDLoc &DL, SDValue Acc, SDValue LHS, SDValue RHS)
Get an expression that implements a partial multiply-subtract reduction.
LLVM_ABI SDValue expandVAArg(SDNode *Node)
Expand the specified ISD::VAARG node as the Legalize pass would.
LLVM_ABI SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
LLVM_ABI bool areNonVolatileConsecutiveStores(StoreSDNode *ST, StoreSDNode *Base, TypeSize Bytes, int Dist) const
Return true if stores are next to each other and can be merged.
LLVM_ABI ConstantRange computeConstantRangeIncludingKnownBits(SDValue Op, bool ForSigned, unsigned Depth=0) const
Combine constant ranges from computeConstantRange() and computeKnownBits().
const SelectionDAGTargetInfo & getSelectionDAGInfo() const
LLVM_ABI SDValue getMaskedHistogram(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
LLVM_ABI SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
LLVM_ABI SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
LLVM_ABI OverflowKind computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const
Determine if the result of the unsigned mul of 2 nodes can overflow.
LLVM_ABI void copyExtraInfo(SDNode *From, SDNode *To)
Copy extra info associated with one node to another.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
LLVM_ABI SDValue getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, MachineMemOperand *MMO)
LLVM_ABI SDValue getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS)
LLVM_ABI SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
LLVM_ABI void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
LLVM_ABI std::pair< SDValue, SDValue > getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, const CallInst *CI)
Lower a memcmp operation into a target library call and return the resulting chain and call result as...
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
LLVM_ABI SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
LLVM_ABI SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
LLVM_ABI OverflowKind computeOverflowForSignedMul(SDValue N0, SDValue N1) const
Determine if the result of the signed mul of 2 nodes can overflow.
LLVM_ABI MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
LLVM_ABI void dump() const
Dump the textual format of this DAG.
LLVM_ABI bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if '(Op & Mask) == Mask'.
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
LLVM_ABI void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
LLVM_ABI void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
LLVM_ABI void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
bool isConstantValueOfAnyType(SDValue N) const
LLVM_ABI bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, UndefPoisonKind Kind=UndefPoisonKind::UndefOrPoison, bool ConsiderFlags=true, unsigned Depth=0) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
SDValue getInsertVectorElt(const SDLoc &DL, SDValue Vec, SDValue Elt, unsigned Idx)
Insert Elt into Vec at offset Idx.
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, Register VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI KnownFPClass computeKnownFPClass(SDValue Op, FPClassTest InterestedClasses, unsigned Depth=0) const
Determine floating-point class information about Op.
LLVM_ABI bool isIdentityElement(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo, unsigned Depth=0) const
Returns true if V is an identity element of Opc with Flags.
LLVM_ABI SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, UndefPoisonKind Kind=UndefPoisonKind::UndefOrPoison, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, Kind can be used to track poison ...
LLVM_ABI bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
LLVM_ABI SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
LLVM_ABI SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
LLVM_ABI std::optional< unsigned > getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
LLVM_ABI SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
LLVM_ABI SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
LLVM_ABI SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
LLVM_ABI std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
LLVM_ABI std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
LLVM_ABI SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
LLVM_ABI SDValue getMaskFromElementCount(const SDLoc &DL, EVT VT, ElementCount Len)
Return a vector with the first 'Len' lanes set to true and remaining lanes set to false.
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
iterator_range< allnodes_iterator > allnodes()
LLVM_ABI SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
LLVM_ABI SDValue WidenVector(const SDValue &N, const SDLoc &DL)
Widen the vector up to the next power of two using INSERT_SUBVECTOR.
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
LLVM_ABI SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the Fra...
ArrayRef< SDDbgValue * > GetDbgValues(const SDNode *SD) const
Get the debug values which reference the given SDNode.
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI OverflowKind computeOverflowForSignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the signed addition of 2 nodes can overflow.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
LLVM_ABI unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
ilist< SDNode >::size_type allnodes_size() const
LLVM_ABI bool isKnownNeverNaN(SDValue Op, const APInt &DemandedElts, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN in...
LLVM_ABI SDValue FoldConstantBuildVector(BuildVectorSDNode *BV, const SDLoc &DL, EVT DstEltVT)
Fold BUILD_VECTOR of constants/undefs to the destination type BUILD_VECTOR of constants/undefs elemen...
LLVM_ABI SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
LLVM_ABI SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
LLVM_ABI SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsCompressing=false)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
const TargetLibraryInfo & getLibInfo() const
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
LLVM_ABI bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if 'Op' is known to be zero in DemandedElts.
LLVM_ABI SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
LLVM_ABI SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
LLVM_ABI SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
LLVM_ABI SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVol, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
LLVM_ABI SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
LLVM_ABI bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void getTopologicallyOrderedNodes(SmallVectorImpl< const SDNode * > &SortedNodes) const
Get all the nodes in their topological order without modifying any states.
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
LLVM_ABI std::pair< SDValue, SDValue > getStrstr(SDValue Chain, const SDLoc &dl, SDValue S0, SDValue S1, const CallInst *CI)
Lower a strstr operation into a target library call and return the resulting chain and call result as...
LLVM_ABI SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
LLVM_ABI OverflowKind computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the unsigned addition of 2 nodes can overflow.
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getErrorMergeValues(ArrayRef< EVT > ResultTypes, SDValue Chain, const SDLoc &dl)
Return poison values for each of ResultTypes, substituting Chain for any result of type MVT::Other,...
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI bool canIgnoreSignBitOfZeroResult(SDValue Op) const
Check if all uses of Op (limited to checking two for compile-time performance) are insensitive to sig...
LLVM_ABI void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, SDValue &N2) const
Swap N1 and N2 if Opcode is a commutative binary opcode and the canonical form expects the opposite o...
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getRegisterMask(const uint32_t *RegMask)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
const SelectionDAGOptions & getSelectionDAGOptions() const
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVM_ABI bool isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth=0) const
Test if the given fp value is known to be an integer power-of-2, either positive or negative.
LLVM_ABI OverflowKind computeOverflowForSignedSub(SDValue N0, SDValue N1) const
Determine if the result of the signed sub of 2 nodes can overflow.
LLVM_ABI void init(MachineFunction &NewMF, const TargetLibraryInfo *LibraryInfo, const LibcallLoweringInfo *LibcallsInfo, UniformityInfo *UA, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs)
Prepare this SelectionDAG to process code in the given MachineFunction.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
LLVM_ABI SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
LLVM_ABI bool isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero=false, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
LLVM_ABI SDValue getDeactivationSymbol(const GlobalValue *GV)
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
LLVM_ABI SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
LLVM_ABI bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
LLVM_ABI std::pair< EVT, EVT > GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, bool *HiIsEmpty) const
Compute the VTs needed for the low/hi parts of a type, dependent on an enveloping VT that has been sp...
LLVM_ABI SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
Fold floating-point operations when all operands are constants and/or undefined.
LLVM_ABI std::optional< ConstantRange > getValidShiftAmountRange(SDValue V, const APInt &DemandedElts, unsigned Depth) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
LLVM_ABI SDValue FoldSymbolOffset(unsigned Opcode, EVT VT, const GlobalAddressSDNode *GA, const SDNode *N2)
LLVM_ABI SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
LLVM_ABI SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
LLVM_ABI SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
DenormalMode getDenormalMode(EVT VT) const
Return the current function's default denormal handling kind for the given floating point type.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, ArrayRef< ISD::NodeType > CandidateBinOps, bool AllowPartials=false)
Match a binop + shuffle pyramid that represents a horizontal reduction over the elements of a vector ...
LLVM_ABI bool isADDLike(SDValue Op, bool NoWrap=false) const
Return true if the specified operand is an ISD::OR or ISD::XOR node that can be treated as an ISD::AD...
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
LLVM_ABI SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
LLVM_ABI void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
LLVM_ABI SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
LLVM_ABI SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
ilist< SDNode >::iterator allnodes_iterator
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
int getMaskElt(unsigned Idx) const
ArrayRef< int > getMask() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
static LLVM_ABI bool isSplatMask(ArrayRef< int > Mask)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
int64_t getFixed() const
Returns the fixed component of the stack.
int64_t getScalable() const
Returns the scalable component of the stack.
This class is used to represent ISD::STORE nodes.
Represent a constant reference to a string, i.e.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Completely target-dependent object reference.
int64_t getOffset() const
unsigned getTargetFlags() const
Provides information about what library functions are available for the current target.
IntegerType * getSizeTType(const Module &M) const
Returns an IntegerType corresponding to size_t.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
std::vector< ArgListEntry > ArgListTy
virtual bool isLegalStoreImmediate(int64_t Value) const
Return true if the specified immediate is legal for the value input of a store instruction.
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
const Triple & getTargetTriple() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const SelectionDAGTargetInfo * getSelectionDAGInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
virtual const TargetLowering * getTargetLowering() const
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, DriverKit, XROS, or bridgeOS).
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
A Use represents the edge between a Value definition and its users.
LLVM_ABI unsigned getOperandNo() const
Return the operand # of this use in its User.
LLVM_ABI void set(Value *Val)
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
This class is used to represent an VP_GATHER node.
This class is used to represent a VP_LOAD node.
This class is used to represent an VP_SCATTER node.
This class is used to represent a VP_STORE node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS*X will result in a value whose quantity matches our ...
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr bool isKnownEven() const
A return value of true indicates we know at compile time that the number of elements (vscale * Min) i...
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI APInt clmulr(const APInt &LHS, const APInt &RHS)
Perform a reversed carry-less multiply.
LLVM_ABI APInt mulhu(const APInt &C1, const APInt &C2)
Performs (2*N)-bit multiplication on zero-extended operands.
LLVM_ABI APInt avgCeilU(const APInt &C1, const APInt &C2)
Compute the ceil of the unsigned average of C1 and C2.
LLVM_ABI APInt avgFloorU(const APInt &C1, const APInt &C2)
Compute the floor of the unsigned average of C1 and C2.
LLVM_ABI APInt pext(const APInt &Val, const APInt &Mask)
Perform a "compress" operation, also known as pext or bext.
LLVM_ABI APInt fshr(const APInt &Hi, const APInt &Lo, const APInt &Shift)
Perform a funnel shift right.
LLVM_ABI APInt mulhs(const APInt &C1, const APInt &C2)
Performs (2*N)-bit multiplication on sign-extended operands.
LLVM_ABI APInt avgSubFloorS(const APInt &C1, const APInt &C2)
Compute floor((C1 - C2) / 2) where C1 and C2 are treated as signed, so the subtraction is performed w...
LLVM_ABI APInt clmul(const APInt &LHS, const APInt &RHS)
Perform a carry-less multiply, also known as XOR multiplication, and return low-bits.
LLVM_ABI APInt pdep(const APInt &Val, const APInt &Mask)
Perform an "expand" operation, also known as pdep or bdep.
APInt abds(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be signed.
LLVM_ABI APInt fshl(const APInt &Hi, const APInt &Lo, const APInt &Shift)
Perform a funnel shift left.
LLVM_ABI APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
LLVM_ABI APInt clmulh(const APInt &LHS, const APInt &RHS)
Perform a carry-less multiply, and return high-bits.
APInt abdu(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be unsigned.
LLVM_ABI APInt avgFloorS(const APInt &C1, const APInt &C2)
Compute the floor of the signed average of C1 and C2.
LLVM_ABI APInt avgCeilS(const APInt &C1, const APInt &C2)
Compute the ceil of the signed average of C1 and C2.
LLVM_ABI APInt avgSubFloorU(const APInt &C1, const APInt &C2)
Compute floor((C1 - C2) / 2) where C1 and C2 are treated as unsigned, so the subtraction is performed...
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, bool isIntegerLike)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
ISD namespace - This namespace contains an enum which represents all of the SelectionDAG node types a...
LLVM_ABI CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
LLVM_ABI bool isBuildVectorOfConstantFPSDNodes(SDValue V)
Return true if the specified value is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
LLVM_ABI bool isConstantSplatVectorAllZeros(SDValue V, bool BuildVectorOnly=false)
Return true if the specified value is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 ...
bool matchUnaryPredicateImpl(SDValue Op, const APInt &DemandedElts, std::function< bool(ConstNodeType *)> Match, bool AllowUndefs=false, bool AllowTruncation=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant BUI...
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ PTRADD
PTRADD represents pointer arithmetic semantics, for targets that opt in using shouldPreservePtrArith(...
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ POISON
POISON - A poison node.
@ PARTIAL_REDUCE_SMLA
PARTIAL_REDUCE_[U|S]MLA(Accumulator, Input1, Input2) The partial reduction nodes sign or zero extend ...
@ ATOMIC_LOAD_FMINIMUMNUM
@ MASK_BEFOREFIRST
Has one mask vector operand.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ AVGSUBFLOORS
AVGSUBFLOORS/AVGSUBFLOORU - Averaging subtract - Subtract two integers using an integer of type i[N+1...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ JUMP_TABLE_DEBUG_INFO
JUMP_TABLE_DEBUG_INFO - Jumptable debug info.
@ BSWAP
Byte Swap and Counting operators.
@ DEACTIVATION_SYMBOL
Untyped node storing deactivation symbol reference (DeactivationSymbolSDNode).
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ PSEUDO_FMIN
PSEUDO_FMIN is strictly equivalent to op0 olt op1 ?
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ VECTOR_SHUFFLE_VAR
VECTOR_SHUFFLE_VAR(VEC, MASK) - Returns VEC permuted by MASK, an integer vector with the same number ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ FMULADD
FMULADD - Performs a * b + c, with, or without, intermediate rounding.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ CLMUL
Carry-less multiplication operations.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SRCVALUE
SRCVALUE - This is a node type that holds a Value* that is used to make reference to a value in the L...
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ TargetIndex
TargetIndex - Like a constant pool entry, but with completely target-dependent semantics.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ VECREDUCE_FMAXIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM nodes do not propagate NaNs and order signed zeroes using the llvm....
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ CTLS
Count leading redundant sign bits.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ MASKED_UDIV
Masked vector arithmetic that returns poison on disabled lanes.
@ LIFETIME_START
This corresponds to the llvm.lifetime.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
@ ATOMIC_LOAD_FMAXIMUMNUM
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ EXPERIMENTAL_VECTOR_HISTOGRAM
Experimental vector histogram intrinsic Operands: Input Chain, Inc, Mask, Base, Index,...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_REPEAT
VECTOR_REPEAT(FIXED_LENGTH_VECTOR) Repeatedly copies the elements of the source fixed-length vector t...
@ SET_FPENV_MEM
Sets the current floating point environment.
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ TRUNCATE_SSAT_S
TRUNCATE_[SU]SAT_[SU] - Truncate for saturated operand [SU] located in middle, prefix for SAT means i...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
LLVM_ABI NodeType getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc)
Given a MinMaxOpc of ISD::(U|S)MIN or ISD::(U|S)MAX, returns the corresponding opcode with the opposi...
LLVM_ABI NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
LLVM_ABI bool isConstantSplatVectorAllOnes(SDValue V, bool BuildVectorOnly=false)
Return true if the specified value is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0...
LLVM_ABI bool isFreezeUndef(SDValue V)
Return true if the specified value is FREEZE(UNDEF).
bool isExtOpcode(unsigned Opcode)
LLVM_ABI NodeType getUnmaskedBinOpOpcode(unsigned MaskedOpc)
Given a MaskedOpc of ISD::MASKED_(U|S)(DIV|REM), returns the unmasked ISD::(U|S)(DIV|REM).
LLVM_ABI bool isVPBinaryOp(unsigned Opcode)
Whether this is a vector-predicated binary operation opcode.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
LLVM_ABI bool allOperandsUndef(SDValue V)
Return true if the value has at least one operand and all operands of the specified value are ISD::UN...
LLVM_ABI std::optional< unsigned > getBaseOpcodeForVP(unsigned Opcode, bool hasFPExcept)
Translate this VP Opcode to its corresponding non-VP Opcode.
bool isBitwiseLogicOp(unsigned Opcode)
Whether this is bitwise logic opcode.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
LLVM_ABI bool isBuildVectorAllOnes(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are ~0 or undef.
LLVM_ABI std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
LLVM_ABI bool isVectorShrinkable(SDValue V, unsigned NewEltSize, bool Signed)
Returns true if the specified value is a vector where all elements can be truncated to the specified ...
unsigned getUnorderedFlavor(CondCode Cond)
This function returns 0 if the condition is always false if an operand is a NaN, 1 if the condition i...
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
LLVM_ABI std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
LLVM_ABI bool matchBinaryPredicate(SDValue LHS, SDValue RHS, const APInt &DemandedElts, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
LLVM_ABI std::optional< unsigned > getVPForBaseOpcode(unsigned Opcode)
Translate this non-VP Opcode to its corresponding VP Opcode.
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
LLVM_ABI NodeType getInverseMinMaxOpcode(unsigned MinMaxOpc)
Given a MinMaxOpc of ISD::(U|S)MIN or ISD::(U|S)MAX, returns ISD::(U|S)MAX and ISD::(U|S)MIN,...
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
LLVM_ABI bool isVPReduction(unsigned Opcode)
Whether this is a vector-predicated reduction opcode.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LLVM_ABI bool isBuildVectorOfConstantSDNodes(SDValue V)
Return true if the specified value is a BUILD_VECTOR node of all ConstantSDNode or undef.
LLVM_ABI NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
LLVM_ABI bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool matchUnaryPredicate(SDValue Op, const APInt &DemandedElts, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTruncation=false)
Hook for matching ConstantSDNode predicate.
LLVM_ABI CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
match_deferred< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
auto m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
LLVM_ABI Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
LLVM_ABI Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
LLVM_ABI Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given e...
bool sd_match(SDValue N, Pattern &&P)
LLVM_ABI unsigned rot(unsigned SrcSignBits, unsigned BitWidth, std::optional< APInt > RotAmt, bool IsRotateRight)
Compute the number of sign bits after rotating a value.
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
NodeAddr< NodeBase * > Node
This is an optimization pass for GlobalISel generic memory operations.
GenericUniformityInfo< SSAContext > UniformityInfo
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
bool operator<(int64_t V1, const APSInt &V2)
LLVM_ABI ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
void fill(R &&Range, T &&Value)
Provide wrappers to std::fill which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI SDValue peekThroughExtractSubvectors(SDValue V)
Return the non-extracted vector source operand of V if it exists.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
LLVM_ABI bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
LLVM_ABI SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs)
If V is a bitwise not, returns the inverted operand.
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
@ Store
The extracted value is stored (ExtractElement only).
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
LLVM_ABI bool getConstantDataArrayInfo(const Value *V, ConstantDataArraySlice &Slice, unsigned ElementSize, uint64_t Offset=0)
Returns true if the value V is a pointer into a ConstantDataArray.
LLVM_ABI bool isOneOrOneSplatFP(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant floating-point value, or a splatted vector of a constant float...
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximum semantics.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
LLVM_ABI bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
auto cast_or_null(const Y &Val)
LLVM_ABI bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
LLVM_ABI bool isMinSignedConstant(SDValue V)
Returns true if V is a constant min signed integer value.
LLVM_ABI ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 maxNum semantics.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
LLVM_ABI bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
auto reverse(ContainerTy &&C)
LLVM_ABI SDValue peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts)
Recursively peek through INSERT_VECTOR_ELT nodes, returning the source vector operand of V,...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_ABI void checkForCycles(const SelectionDAG *DAG, bool force=false)
void sort(IteratorTy Start, IteratorTy End)
LLVM_READONLY APFloat minimumnum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimumNumber semantics.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI SDValue peekThroughTruncates(SDValue V)
Return the non-truncated source operand of V if it exists.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr std::underlying_type_t< Enum > to_underlying(Enum E)
Returns underlying integer value of an enum.
LLVM_ABI ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
LLVM_ABI SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
LLVM_ABI MDNode * getMemCacheHint(const Instruction &I, unsigned OperandNo=0)
Return the hint payload for operand OperandNo of I, or nullptr if absent.
CodeGenOptLevel
Code generation optimization level.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
bool includesPoison(UndefPoisonKind Kind)
Returns true if Kind includes the Poison bit.
LLVM_ABI bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
bool includesUndef(UndefPoisonKind Kind)
Returns true if Kind includes the Undef bit.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 minNum semantics.
@ Sub
Subtraction of integers.
@ Fast
Assign the register banks as fast as possible (default).
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
LLVM_ABI bool isNullConstantOrUndef(SDValue V)
Returns true if V is a constant integer zero or an UNDEF node.
LLVM_ABI bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
OutputIt copy(R &&Range, OutputIt Out)
constexpr unsigned BitWidth
LLVM_ABI bool funcReturnsFirstArgOfCall(const CallInst &CI)
Returns true if the parent of CI returns CI's first argument after calling CI.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool isZeroOrZeroSplat(SDValue N, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
UndefPoisonKind
Enumeration to track whether we are interested in Undef, Poison, or both.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
LLVM_ABI bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
LLVM_ABI bool isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs=false)
Return true if the value is a constant (+/-)0.0 floating-point value or a splatted vector thereof (wi...
bool equal(L &&LRange, R &&RRange)
Wrapper function around std::equal to detect if pair-wise elements between two ranges are the same.
LLVM_ABI void computeKnownBitsFromRangeMetadata(const MDNode &Ranges, KnownBits &Known)
Compute known bits from the range metadata.
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimum semantics.
LLVM_READONLY APFloat maximumnum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximumNumber semantics.
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
MDNode * TBAAStruct
The tag for type-based alias analysis (tbaa struct).
MDNode * TBAA
The tag for type-based alias analysis.
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
static constexpr Align fromLog2(unsigned Log2Value)
Returns an alignment of 1 << Log2Value bytes.
Represents offset+length into a ConstantDataArray.
uint64_t Length
Length of the slice.
uint64_t Offset
Slice starts at this Offset.
void move(uint64_t Delta)
Moves the Offset and adjusts Length accordingly.
const ConstantDataArray * Array
ConstantDataArray pointer.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
intptr_t getRawBits() const
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
EVT changeElementCount(LLVMContext &Context, ElementCount EC) const
Return a VT for a type whose attributes match ourselves with the exception of the element count that ...
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
static LLVM_ABI KnownBits mulhu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from zero-extended multiply-hi.
static LLVM_ABI KnownBits smax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smax(LHS, RHS).
bool isNonNegative() const
Returns true if this value is known to be non-negative.
bool isZero() const
Returns true if value is all zero.
static LLVM_ABI KnownBits usub_sat(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from llvm.usub.sat(LHS, RHS)
static LLVM_ABI KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
static LLVM_ABI KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
static LLVM_ABI std::optional< bool > ne(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_NE result.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
KnownBits byteSwap() const
static LLVM_ABI KnownBits fshl(const KnownBits &LHS, const KnownBits &RHS, const APInt &Amt)
Compute known bits for fshl(LHS, RHS, Amt).
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
void setAllZero()
Make all bits known to be zero and discard any previous information.
KnownBits reverseBits() const
KnownBits concat(const KnownBits &Lo) const
Concatenate the bits from Lo onto the bottom of *this.
unsigned getBitWidth() const
Get the bit width of this value.
static LLVM_ABI KnownBits umax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umax(LHS, RHS).
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
static KnownBits add(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false, bool SelfAdd=false)
Compute knownbits resulting from addition of LHS and RHS.
static LLVM_ABI KnownBits lshr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for lshr(LHS, RHS).
bool isNonZero() const
Returns true if this value is known to be non-zero.
static LLVM_ABI KnownBits abdu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for abdu(LHS, RHS).
KnownBits extractBits(unsigned NumBits, unsigned BitPosition) const
Return a subset of the known bits from [bitPosition,bitPosition+numBits).
static LLVM_ABI KnownBits pdep(const KnownBits &Val, const KnownBits &Mask)
Compute known bits for pdep(Val, Mask).
static LLVM_ABI KnownBits avgFloorU(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgFloorU.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static LLVM_ABI KnownBits computeForSubBorrow(const KnownBits &LHS, KnownBits RHS, const KnownBits &Borrow)
Compute known bits results from subtracting RHS from LHS with 1-bit Borrow.
KnownBits zextOrTrunc(unsigned BitWidth) const
Return known bits for a zero extension or truncation of the value we're tracking.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
static LLVM_ABI KnownBits fshr(const KnownBits &LHS, const KnownBits &RHS, const APInt &Amt)
Compute known bits for fshr(LHS, RHS, Amt).
static LLVM_ABI KnownBits abds(KnownBits LHS, KnownBits RHS)
Compute known bits for abds(LHS, RHS).
static LLVM_ABI KnownBits smin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smin(LHS, RHS).
static LLVM_ABI KnownBits mulhs(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from sign-extended multiply-hi.
static LLVM_ABI KnownBits srem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for srem(LHS, RHS).
static LLVM_ABI KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
bool isStrictlyPositive() const
Returns true if this value is known to be positive.
static LLVM_ABI KnownBits sdiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for sdiv(LHS, RHS).
static LLVM_ABI KnownBits avgFloorS(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgFloorS.
static bool haveNoCommonBitsSet(const KnownBits &LHS, const KnownBits &RHS)
Return true if LHS and RHS have no common bits set.
bool isNegative() const
Returns true if this value is known to be negative.
static LLVM_ABI KnownBits computeForAddCarry(const KnownBits &LHS, const KnownBits &RHS, const KnownBits &Carry)
Compute known bits resulting from adding LHS, RHS and a 1-bit Carry.
static KnownBits sub(const KnownBits &LHS, const KnownBits &RHS, bool NSW=false, bool NUW=false)
Compute knownbits resulting from subtraction of LHS and RHS.
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
static LLVM_ABI KnownBits avgCeilU(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgCeilU.
static LLVM_ABI KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
KnownBits anyext(unsigned BitWidth) const
Return known bits for an "any" extension of the value we're tracking, where we don't know anything ab...
static LLVM_ABI KnownBits clmul(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for clmul(LHS, RHS).
LLVM_ABI KnownBits abs(bool IntMinIsPoison=false) const
Compute known bits for the absolute value.
static LLVM_ABI KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
static LLVM_ABI KnownBits umin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umin(LHS, RHS).
static LLVM_ABI KnownBits pext(const KnownBits &Val, const KnownBits &Mask)
Compute known bits for pext(Val, Mask).
static LLVM_ABI KnownBits avgCeilS(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from APIntOps::avgCeilS.
This class contains a discriminated union of information about pointers in memory operands,...
LLVM_ABI bool isDereferenceable(unsigned Size, LLVMContext &C, const DataLayout &DL) const
Return true if memory region [V, V+Offset+Size) is known to be dereferenceable.
LLVM_ABI unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
PointerUnion< const Value *, const PseudoSourceValue * > V
This is the IR pointer value for the access, or it is null if unknown.
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, bool IsZeroMemset, bool IsVolatile)
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc=false)
static MemOp Move(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile)
static StringRef getLibcallImplName(RTLIB::LibcallImpl CallImpl)
Get the libcall routine name for the specified libcall implementation.
These are IR-level optimization flags that may be propagated to SDNodes.
static LLVM_ABI bool isEqual(const KeyTy &Key, const SDNode &N)
The key SelectionDAG uniques SDNodes by.
void AddPointer(const void *P)
SmallVector< SDValue, 0 > OpStorage
Backs Ops when the key is built from a node; empty otherwise.
SDNodeKey(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
DAGUpdateListener *const Next
virtual void NodeDeleted(SDNode *N, SDNode *E)
The node N that was deleted and, if E is not null, an equivalent node E that replaced it.
virtual void NodeInserted(SDNode *N)
The node N that was inserted.
virtual void NodeUpdated(SDNode *N)
The node N that was updated.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)