79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsBPF.h"
82#include "llvm/IR/IntrinsicsWebAssembly.h"
118#define DEBUG_TYPE "isel"
126 cl::desc(
"Insert the experimental `assertalign` node."),
131 cl::desc(
"Generate low-precision inline sequences "
132 "for some float libcalls"),
138 cl::desc(
"Set the case probability threshold for peeling the case from a "
139 "switch statement. A value greater than 100 will void this "
162 std::optional<CallingConv::ID>
CC);
173 std::optional<CallingConv::ID>
CC = std::nullopt,
174 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
178 PartVT, ValueVT,
CC))
196 unsigned RoundBits = PartBits * RoundParts;
197 EVT RoundVT = RoundBits == ValueBits ?
203 if (RoundParts > 2) {
207 PartVT, HalfVT,
V, InChain);
220 unsigned OddParts =
NumParts - RoundParts;
223 OddVT,
V, InChain,
CC);
239 assert(ValueVT ==
EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
250 !PartVT.
isVector() &&
"Unexpected split");
262 if (PartEVT == ValueVT)
266 ValueVT.
bitsLT(PartEVT)) {
279 if (ValueVT.
bitsLT(PartEVT)) {
284 Val = DAG.
getNode(*AssertOp,
DL, PartEVT, Val,
299 llvm::Attribute::StrictFP)) {
301 DAG.
getVTList(ValueVT, MVT::Other), InChain, Val,
313 if (PartEVT == MVT::x86mmx && ValueVT.
isInteger() &&
314 ValueVT.
bitsLT(PartEVT)) {
323 const Twine &ErrMsg) {
326 return Ctx.emitError(ErrMsg);
329 if (CI->isInlineAsm()) {
331 *CI, ErrMsg +
", possible invalid constraint for vector type"));
334 return Ctx.emitError(
I, ErrMsg);
340 MVT RegisterVT,
MVT PartVT) {
341 if (NumRegs ==
NumParts && RegisterVT == PartVT)
345 Twine(
"Part count doesn't match vector breakdown! ") +
Twine(NumRegs) +
347 " RegisterVT, " + PartVT.
getString() +
" PartVT");
359 std::optional<CallingConv::ID> CallConv) {
362 const bool IsABIRegCopy = CallConv.has_value();
371 unsigned NumIntermediates;
375 NumRegs =
TLI.getVectorTypeBreakdownForCallingConv(
376 *DAG.
getContext(), *CallConv, ValueVT, IntermediateVT,
377 NumIntermediates, RegisterVT);
380 TLI.getVectorTypeBreakdown(*DAG.
getContext(), ValueVT, IntermediateVT,
381 NumIntermediates, RegisterVT);
387 "Part type sizes don't match!");
394 for (
unsigned i = 0; i !=
NumParts; ++i)
396 V, InChain, CallConv);
401 "Must expand into a divisible number of parts!");
402 unsigned Factor =
NumParts / NumIntermediates;
403 for (
unsigned i = 0; i != NumIntermediates; ++i)
405 IntermediateVT,
V, InChain, CallConv);
420 DL, BuiltVectorTy,
Ops);
426 if (PartEVT == ValueVT)
442 "Cannot narrow, it would be a lossy transformation");
448 if (PartEVT == ValueVT)
465 TLI.isTypeLegal(ValueVT))
473 }
else if (ValueVT.
bitsLT(PartEVT)) {
482 *DAG.
getContext(),
V,
"non-trivial scalar-to-vector conversion");
513 std::optional<CallingConv::ID> CallConv);
521 std::optional<CallingConv::ID> CallConv = std::nullopt,
525 if (
TLI.splitValueIntoRegisterParts(DAG,
DL, Val, Parts,
NumParts, PartVT,
537 "Copying to an illegal type!");
543 EVT PartEVT = PartVT;
544 if (PartEVT == ValueVT) {
565 "Unknown mismatch!");
568 if (PartVT == MVT::x86mmx)
579 "Unknown mismatch!");
582 if (PartVT == MVT::x86mmx)
589 "Failed to tile the value with PartVT!");
592 if (PartEVT != ValueVT) {
594 "scalar-to-vector conversion failed");
606 "Do not know what to expand to!");
608 unsigned RoundBits = RoundParts * PartBits;
609 unsigned OddParts =
NumParts - RoundParts;
618 std::reverse(Parts + RoundParts, Parts +
NumParts);
632 for (
unsigned StepSize =
NumParts; StepSize > 1; StepSize /= 2) {
633 for (
unsigned i = 0; i <
NumParts; i += StepSize) {
634 unsigned ThisBits = StepSize * PartBits / 2;
637 SDValue &Part1 = Parts[i+StepSize/2];
644 if (ThisBits == PartBits && ThisVT != PartVT) {
652 std::reverse(Parts, Parts + OrigNumParts);
674 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
676 "Cannot widen to illegal type");
680 }
else if (PartEVT != ValueEVT) {
695 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
706 std::optional<CallingConv::ID> CallConv) {
710 const bool IsABIRegCopy = CallConv.has_value();
713 EVT PartEVT = PartVT;
714 if (PartEVT == ValueVT) {
760 "lossy conversion of vector to scalar type");
775 unsigned NumIntermediates;
778 NumRegs =
TLI.getVectorTypeBreakdownForCallingConv(
779 *DAG.
getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
783 TLI.getVectorTypeBreakdown(*DAG.
getContext(), ValueVT, IntermediateVT,
784 NumIntermediates, RegisterVT);
789 "Mixing scalable and fixed vectors when copying in parts");
791 std::optional<ElementCount> DestEltCnt;
801 if (ValueVT == BuiltVectorTy) {
825 for (
unsigned i = 0; i != NumIntermediates; ++i) {
843 for (
unsigned i = 0; i !=
NumParts; ++i)
848 assert(NumIntermediates != 0 &&
"division by zero");
850 "Must expand into a divisible number of parts!");
851 unsigned Factor =
NumParts / NumIntermediates;
852 for (
unsigned i = 0; i != NumIntermediates; ++i)
860 if (
I.hasOperandBundlesOtherThan(AllowedBundles)) {
864 for (
unsigned i = 0, e =
I.getNumOperandBundles(); i != e; ++i) {
867 OS << LS << U.getTagName();
870 Twine(
"cannot lower ", Name)
876 EVT valuevt, std::optional<CallingConv::ID>
CC)
882 std::optional<CallingConv::ID>
CC) {
890 ?
TLI.getNumRegistersForCallingConv(Context, *
CC, ValueVT)
891 :
TLI.getNumRegisters(Context, ValueVT);
894 ?
TLI.getRegisterTypeForCallingConv(Context, *
CC, ValueVT)
895 :
TLI.getRegisterType(Context, ValueVT);
896 for (
unsigned i = 0; i != NumRegs; ++i)
897 Regs.push_back(Reg + i);
898 RegVTs.push_back(RegisterVT);
900 Reg = Reg.id() + NumRegs;
922 ?
TLI.getRegisterTypeForCallingConv(
927 for (
unsigned i = 0; i != NumRegs; ++i) {
933 *Glue =
P.getValue(2);
936 Chain =
P.getValue(1);
964 EVT FromVT(MVT::Other);
968 }
else if (NumSignBits > 1) {
976 assert(FromVT != MVT::Other);
982 RegisterVT, ValueVT, V, Chain,
CallConv);
998 unsigned NumRegs =
Regs.size();
1004 ?
TLI.getRegisterTypeForCallingConv(
1019 for (
unsigned i = 0; i != NumRegs; ++i) {
1031 if (NumRegs == 1 || Glue)
1042 Chain = Chains[NumRegs-1];
1048 unsigned MatchingIdx,
const SDLoc &dl,
1050 std::vector<SDValue> &
Ops)
const {
1055 Flag.setMatchingOp(MatchingIdx);
1056 else if (!
Regs.empty() &&
Regs.front().isVirtual()) {
1064 Flag.setRegClass(RC->
getID());
1075 "No 1:1 mapping from clobbers to regs?");
1076 Register SP =
TLI.getStackPointerRegisterToSaveRestore();
1078 for (
unsigned I = 0, E =
ValueVTs.size();
I != E; ++
I) {
1083 "If we clobbered the stack pointer, MFI should know about it.");
1092 for (
unsigned i = 0; i != NumRegs; ++i) {
1093 assert(Reg <
Regs.size() &&
"Mismatch in # registers expected");
1105 unsigned RegCount = std::get<0>(CountAndVT);
1106 MVT RegisterVT = std::get<1>(CountAndVT);
1124 SL->init(
DAG.getTargetLoweringInfo(), TM,
DAG.getDataLayout());
1126 *
DAG.getMachineFunction().getFunction().getParent());
1127 CanDescribeGlobalAddressInLocationList =
1133 UnusedArgNodeMap.clear();
1135 PendingExports.clear();
1136 PendingConstrainedFP.clear();
1137 PendingConstrainedFPStrict.clear();
1145 DanglingDebugInfoMap.clear();
1152 if (Pending.
empty())
1158 unsigned i = 0, e = Pending.
size();
1159 for (; i != e; ++i) {
1161 if (Pending[i].
getNode()->getOperand(0) == Root)
1169 if (Pending.
size() == 1)
1196 if (!PendingConstrainedFPStrict.empty()) {
1197 assert(PendingConstrainedFP.empty());
1198 updateRoot(PendingConstrainedFPStrict);
1211 if (!PendingConstrainedFP.empty()) {
1212 assert(PendingConstrainedFPStrict.empty());
1213 updateRoot(PendingConstrainedFP);
1217 return DAG.getRoot();
1225 PendingConstrainedFP.size() +
1226 PendingConstrainedFPStrict.size());
1228 PendingConstrainedFP.end());
1229 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1230 PendingConstrainedFPStrict.end());
1231 PendingConstrainedFP.clear();
1232 PendingConstrainedFPStrict.clear();
1239 PendingExports.append(PendingConstrainedFPStrict.begin(),
1240 PendingConstrainedFPStrict.end());
1241 PendingConstrainedFPStrict.clear();
1242 return updateRoot(PendingExports);
1249 assert(Variable &&
"Missing variable");
1256 <<
"dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1272 if (IsParameter && FINode) {
1274 SDV =
DAG.getFrameIndexDbgValue(Variable,
Expression, FINode->getIndex(),
1275 true,
DL, SDNodeOrder);
1280 FuncArgumentDbgValueKind::Declare,
N);
1283 SDV =
DAG.getDbgValue(Variable,
Expression,
N.getNode(),
N.getResNo(),
1284 true,
DL, SDNodeOrder);
1286 DAG.AddDbgValue(SDV, IsParameter);
1291 FuncArgumentDbgValueKind::Declare,
N)) {
1293 <<
" (could not emit func-arg dbg_value)\n");
1304 for (
auto It = FnVarLocs->locs_begin(&
I),
End = FnVarLocs->locs_end(&
I);
1306 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1308 if (It->Values.isKillLocation(It->Expr)) {
1314 It->Values.hasArgList())) {
1317 FnVarLocs->getDILocalVariable(It->VariableID),
1318 It->Expr, Vals.
size() > 1, It->DL, SDNodeOrder);
1331 bool SkipDbgVariableRecords =
DAG.getFunctionVarLocs();
1334 for (
DbgRecord &DR :
I.getDbgRecordRange()) {
1336 assert(DLR->getLabel() &&
"Missing label");
1338 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);
1339 DAG.AddDbgLabel(SDV);
1343 if (SkipDbgVariableRecords)
1351 if (
FuncInfo.PreprocessedDVRDeclares.contains(&DVR))
1353 LLVM_DEBUG(
dbgs() <<
"SelectionDAG visiting dbg_declare: " << DVR
1379 SDNodeOrder, IsVariadic)) {
1390 if (
I.isTerminator()) {
1391 HandlePHINodesInSuccessorBlocks(
I.getParent());
1398 bool NodeInserted =
false;
1399 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1400 MDNode *PCSectionsMD =
I.getMetadata(LLVMContext::MD_pcsections);
1401 MDNode *MMRA =
I.getMetadata(LLVMContext::MD_mmra);
1402 if (PCSectionsMD || MMRA) {
1403 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1404 DAG, [&](
SDNode *) { NodeInserted =
true; });
1414 if (PCSectionsMD || MMRA) {
1415 auto It = NodeMap.find(&
I);
1416 if (It != NodeMap.end()) {
1418 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1420 DAG.addMMRAMetadata(It->second.getNode(), MMRA);
1421 }
else if (NodeInserted) {
1424 errs() <<
"warning: loosing !pcsections and/or !mmra metadata ["
1425 <<
I.getModule()->getName() <<
"]\n";
1434void SelectionDAGBuilder::visitPHI(
const PHINode &) {
1444#define HANDLE_INST(NUM, OPCODE, CLASS) \
1445 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1446#include "llvm/IR/Instruction.def"
1483 DanglingDebugInfoMap[
Values[0]].emplace_back(Var, Expr,
DL, Order);
1488 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1489 DIVariable *DanglingVariable = DDI.getVariable();
1491 if (DanglingVariable == Variable && Expr->
fragmentsOverlap(DanglingExpr)) {
1493 << printDDI(
nullptr, DDI) <<
"\n");
1499 for (
auto &DDIMI : DanglingDebugInfoMap) {
1500 DanglingDebugInfoVector &DDIV = DDIMI.second;
1504 for (
auto &DDI : DDIV)
1505 if (isMatchingDbgValue(DDI))
1508 erase_if(DDIV, isMatchingDbgValue);
1516 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1517 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1520 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1521 for (
auto &DDI : DDIV) {
1523 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1526 assert(Variable->isValidLocationForIntrinsic(
DL) &&
1527 "Expected inlined-at fields to agree");
1537 if (!EmitFuncArgumentDbgValue(V, Variable, Expr,
DL,
1538 FuncArgumentDbgValueKind::Value, Val)) {
1540 << printDDI(V, DDI) <<
"\n");
1547 <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to "
1548 << ValSDNodeOrder <<
"\n");
1549 SDV = getDbgValue(Val, Variable, Expr,
DL,
1550 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1551 DAG.AddDbgValue(SDV,
false);
1555 <<
" in EmitFuncArgumentDbgValue\n");
1557 LLVM_DEBUG(
dbgs() <<
"Dropping debug info for " << printDDI(V, DDI)
1561 DAG.getConstantDbgValue(Variable, Expr, Poison,
DL, DbgSDNodeOrder);
1562 DAG.AddDbgValue(SDV,
false);
1589 DanglingDebugInfo &DDI) {
1594 const Value *OrigV = V;
1598 unsigned SDOrder = DDI.getSDNodeOrder();
1607 auto HandleGlobalAddress = [&] {
1615 false,
DL, SDOrder,
false);
1616 DAG.AddDbgValue(SDV,
false);
1622 HandleGlobalAddress())
1644 if (!AdditionalValues.
empty())
1653 HandleGlobalAddress()) {
1655 dbgs() <<
"Salvaged debug location info for:\n " << *Var <<
"\n"
1656 << *OrigV <<
"\nBy stripping back to:\n " << *V <<
"\n");
1664 assert(OrigV &&
"V shouldn't be null");
1666 auto *SDV =
DAG.getConstantDbgValue(Var, Expr, Poison,
DL, SDOrder);
1667 DAG.AddDbgValue(SDV,
false);
1669 << printDDI(OrigV, DDI) <<
"\n");
1686 unsigned Order,
bool IsVariadic) {
1691 if (visitEntryValueDbgValue(
Values, Var, Expr, DbgLoc))
1706 if (CE->getOpcode() == Instruction::IntToPtr) {
1716 if (CanDescribeGlobalAddressInLocationList)
1718 V, Expr, OpIdx,
DAG.getMachineFunction())) {
1737 N = UnusedArgNodeMap[V];
1742 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1743 FuncArgumentDbgValueKind::Value,
N))
1770 bool IsParamOfFunc =
1778 auto VMI =
FuncInfo.ValueMap.find(V);
1779 if (VMI !=
FuncInfo.ValueMap.end()) {
1784 V->getType(), std::nullopt);
1790 unsigned BitsToDescribe = 0;
1792 BitsToDescribe = *VarSize;
1794 BitsToDescribe =
Fragment->SizeInBits;
1797 if (
Offset >= BitsToDescribe)
1800 unsigned RegisterSize = RegAndSize.second;
1801 unsigned FragmentSize = (
Offset + RegisterSize > BitsToDescribe)
1802 ? BitsToDescribe -
Offset
1805 Expr,
Offset, FragmentSize);
1809 Var, *FragmentExpr, RegAndSize.first,
false, DbgLoc, Order);
1810 DAG.AddDbgValue(SDV,
false);
1826 DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1827 false, DbgLoc, Order, IsVariadic);
1828 DAG.AddDbgValue(SDV,
false);
1834 for (
auto &Pair : DanglingDebugInfoMap)
1835 for (
auto &DDI : Pair.second)
1843 auto It =
FuncInfo.ValueMap.find(V);
1846 if (It !=
FuncInfo.ValueMap.end()) {
1850 DAG.getDataLayout(), InReg, Ty,
1867 if (
N.getNode())
return N;
1882 if (V->getType()->isVoidTy())
1887 V->getType(), ValueVTs);
1888 if (ValueVTs.
empty())
1922 EVT VT =
TLI.getValueType(
DAG.getDataLayout(), V->getType(),
true);
1939 return DAG.getSplatBuildVector(
1942 return DAG.getConstant(*CI,
DL, VT);
1954 getValue(CPA->getAddrDiscriminator()),
1955 getValue(CPA->getDiscriminator()));
1971 visit(CE->getOpcode(), *CE);
1973 assert(N1.
getNode() &&
"visit didn't populate the NodeMap!");
1979 for (
const Use &U :
C->operands()) {
1985 for (
unsigned i = 0, e = Val->
getNumValues(); i != e; ++i)
1986 Constants.push_back(
SDValue(Val, i));
1995 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {
1999 for (
unsigned i = 0, e = Val->
getNumValues(); i != e; ++i)
2008 if (
C->getType()->isStructTy() ||
C->getType()->isArrayTy()) {
2010 "Unknown struct or array constant!");
2014 unsigned NumElts = ValueVTs.
size();
2018 for (
unsigned i = 0; i != NumElts; ++i) {
2019 EVT EltVT = ValueVTs[i];
2021 Constants[i] =
DAG.getUNDEF(EltVT);
2032 return DAG.getBlockAddress(BA, VT);
2035 return getValue(Equiv->getGlobalValue());
2040 if (VT == MVT::aarch64svcount) {
2041 assert(
C->isNullValue() &&
"Can only zero this target type!");
2047 assert(
C->isNullValue() &&
"Can only zero this target type!");
2057 if (VT == MVT::externref || VT == MVT::funcref) {
2058 assert(
C->isNullValue() &&
"Can only zero this target type!");
2061 Intrinsic::ID IID = VT == MVT::externref ? Intrinsic::wasm_ref_null_extern
2062 : Intrinsic::wasm_ref_null_func;
2074 for (
unsigned i = 0; i != NumElements; ++i)
2101 return DAG.getFrameIndex(
2102 SI->second,
TLI.getValueType(
DAG.getDataLayout(), AI->getType()));
2109 Inst->getType(), std::nullopt);
2123void SelectionDAGBuilder::visitCatchPad(
const CatchPadInst &
I) {
2136 if (IsMSVCCXX || IsCoreCLR)
2142 MachineBasicBlock *TargetMBB =
FuncInfo.getMBB(
I.getSuccessor());
2143 FuncInfo.MBB->addSuccessor(TargetMBB);
2150 if (TargetMBB != NextBlock(
FuncInfo.MBB) ||
2159 DAG.getMachineFunction().setHasEHContTarget(
true);
2165 Value *ParentPad =
I.getCatchSwitchParentPad();
2168 SuccessorColor = &
FuncInfo.Fn->getEntryBlock();
2171 assert(SuccessorColor &&
"No parent funclet for catchret!");
2172 MachineBasicBlock *SuccessorColorMBB =
FuncInfo.getMBB(SuccessorColor);
2173 assert(SuccessorColorMBB &&
"No MBB for SuccessorColor!");
2178 DAG.getBasicBlock(SuccessorColorMBB));
2182void SelectionDAGBuilder::visitCleanupPad(
const CleanupPadInst &CPI) {
2188 FuncInfo.MBB->setIsEHFuncletEntry();
2189 FuncInfo.MBB->setIsCleanupFuncletEntry();
2219 UnwindDests.emplace_back(FuncInfo.
getMBB(EHPadBB), Prob);
2225 UnwindDests.emplace_back(FuncInfo.
getMBB(EHPadBB), Prob);
2226 UnwindDests.back().first->setIsEHScopeEntry();
2228 if (!IsWasmCXX && !IsWasmD)
2229 UnwindDests.back().first->setIsEHFuncletEntry();
2233 for (
const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2234 UnwindDests.emplace_back(FuncInfo.
getMBB(CatchPadBB), Prob);
2236 if (IsMSVCCXX || IsCoreCLR)
2237 UnwindDests.back().first->setIsEHFuncletEntry();
2239 UnwindDests.back().first->setIsEHScopeEntry();
2241 NewEHPadBB = CatchSwitch->getUnwindDest();
2247 if (BPI && NewEHPadBB)
2249 EHPadBB = NewEHPadBB;
2256 auto UnwindDest =
I.getUnwindDest();
2257 BranchProbabilityInfo *BPI =
FuncInfo.BPI;
2258 BranchProbability UnwindDestProb =
2263 for (
auto &UnwindDest : UnwindDests) {
2264 UnwindDest.first->setIsEHPad();
2265 addSuccessorWithProb(
FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2267 FuncInfo.MBB->normalizeSuccProbs();
2270 MachineBasicBlock *CleanupPadMBB =
2271 FuncInfo.getMBB(
I.getCleanupPad()->getParent());
2277void SelectionDAGBuilder::visitCatchSwitch(
const CatchSwitchInst &CSI) {
2281void SelectionDAGBuilder::visitRet(
const ReturnInst &
I) {
2282 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
2283 auto &
DL =
DAG.getDataLayout();
2295 if (
I.getParent()->getTerminatingDeoptimizeCall()) {
2309 Type *RetTy =
I.getOperand(0)->getType();
2310 Align BaseAlign =
DL.getPrefTypeAlign(RetTy);
2316 SmallVector<uint64_t, 4>
Offsets;
2318 unsigned NumValues = ValueVTs.
size();
2321 for (
unsigned i = 0; i != NumValues; ++i) {
2328 if (MemVTs[i] != ValueVTs[i])
2330 Chains[i] =
DAG.getStore(
2338 MVT::Other, Chains);
2339 }
else if (
I.getNumOperands() != 0) {
2342 unsigned NumValues =
Types.size();
2346 const Function *
F =
I.getParent()->getParent();
2349 I.getOperand(0)->getType(),
F->getCallingConv(),
2353 if (
F->getAttributes().hasRetAttr(Attribute::SExt))
2355 else if (
F->getAttributes().hasRetAttr(Attribute::ZExt))
2358 LLVMContext &
Context =
F->getContext();
2359 bool RetInReg =
F->getAttributes().hasRetAttr(Attribute::InReg);
2361 for (
unsigned j = 0;
j != NumValues; ++
j) {
2377 ISD::ArgFlagsTy
Flags = ISD::ArgFlagsTy();
2381 if (
I.getOperand(0)->getType()->isPointerTy()) {
2383 Flags.setPointerAddrSpace(
2387 if (NeedsRegBlock) {
2388 Flags.setInConsecutiveRegs();
2389 if (j == NumValues - 1)
2390 Flags.setInConsecutiveRegsLast();
2398 else if (
F->getAttributes().hasRetAttr(Attribute::NoExt))
2401 for (
unsigned i = 0; i <
NumParts; ++i) {
2404 VT, Types[j], 0, 0));
2414 const Function *
F =
I.getParent()->getParent();
2416 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2418 ISD::ArgFlagsTy
Flags = ISD::ArgFlagsTy();
2419 Flags.setSwiftError();
2431 bool isVarArg =
DAG.getMachineFunction().getFunction().isVarArg();
2433 DAG.getMachineFunction().getFunction().getCallingConv();
2434 Chain =
DAG.getTargetLoweringInfo().LowerReturn(
2439 "LowerReturn didn't return a valid chain!");
2450 if (V->getType()->isEmptyTy())
2453 auto VMI =
FuncInfo.ValueMap.find(V);
2454 if (VMI !=
FuncInfo.ValueMap.end()) {
2456 "Unused value assigned virtual registers!");
2469 if (
FuncInfo.isExportedInst(V))
return;
2481 if (VI->getParent() == FromBB)
2507 const BasicBlock *SrcBB = Src->getBasicBlock();
2508 const BasicBlock *DstBB = Dst->getBasicBlock();
2512 auto SuccSize = std::max<uint32_t>(
succ_size(SrcBB), 1);
2522 Src->addSuccessorWithoutProb(Dst);
2525 Prob = getEdgeProbability(Src, Dst);
2526 Src->addSuccessor(Dst, Prob);
2532 return I->getParent() == BB;
2556 if (CurBB == SwitchBB ||
2562 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2567 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2569 if (FC->hasNoNaNs() ||
2577 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1),
nullptr,
2579 SL->SwitchCases.push_back(CB);
2588 SL->SwitchCases.push_back(CB);
2596 unsigned Depth = 0) {
2605 if (Necessary !=
nullptr) {
2608 if (Necessary->contains(
I))
2616 for (
unsigned OpIdx = 0,
E =
I->getNumOperands(); OpIdx <
E; ++OpIdx)
2636 if (BPI !=
nullptr) {
2644 std::optional<bool> Likely;
2647 else if (BPI->
isEdgeHot(
I.getParent(), IfFalse))
2651 if (
Opc == (*Likely ? Instruction::And : Instruction::Or))
2663 if (CostThresh <= 0)
2684 Value *BrCond =
I.getCondition();
2685 auto ShouldCountInsn = [&RhsDeps, &BrCond](
const Instruction *Ins) {
2686 for (
const auto *U : Ins->users()) {
2689 if (UIns != BrCond && !RhsDeps.
contains(UIns))
2702 for (
unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2704 for (
const auto &InsPair : RhsDeps) {
2705 if (!ShouldCountInsn(InsPair.first)) {
2706 ToDrop = InsPair.first;
2710 if (ToDrop ==
nullptr)
2712 RhsDeps.erase(ToDrop);
2715 for (
const auto &InsPair : RhsDeps) {
2720 CostOfIncluding +=
TTI->getInstructionCost(
2723 if (CostOfIncluding > CostThresh)
2749 const Value *BOpOp0, *BOpOp1;
2763 if (BOpc == Instruction::And)
2764 BOpc = Instruction::Or;
2765 else if (BOpc == Instruction::Or)
2766 BOpc = Instruction::And;
2772 bool BOpIsInOrAndTree = BOpc && BOpc ==
Opc && BOp->
hasOneUse();
2777 TProb, FProb, InvertCond);
2787 if (
Opc == Instruction::Or) {
2808 auto NewTrueProb = TProb / 2;
2809 auto NewFalseProb = TProb / 2 + FProb;
2812 NewFalseProb, InvertCond);
2819 Probs[1], InvertCond);
2821 assert(
Opc == Instruction::And &&
"Unknown merge op!");
2841 auto NewTrueProb = TProb + FProb / 2;
2842 auto NewFalseProb = FProb / 2;
2845 NewFalseProb, InvertCond);
2852 Probs[1], InvertCond);
2861 if (Cases.size() != 2)
return true;
2865 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2866 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2867 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2868 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2874 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2875 Cases[0].
CC == Cases[1].
CC &&
2878 if (Cases[0].
CC ==
ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2880 if (Cases[0].
CC ==
ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2887void SelectionDAGBuilder::visitUncondBr(
const UncondBrInst &
I) {
2897 if (Succ0MBB != NextBlock(BrMBB) ||
2906void SelectionDAGBuilder::visitCondBr(
const CondBrInst &
I) {
2907 MachineBasicBlock *BrMBB =
FuncInfo.MBB;
2909 MachineBasicBlock *Succ0MBB =
FuncInfo.getMBB(
I.getSuccessor(0));
2913 const Value *CondVal =
I.getCondition();
2914 MachineBasicBlock *Succ1MBB =
FuncInfo.getMBB(
I.getSuccessor(1));
2933 bool IsUnpredictable =
I.hasMetadata(LLVMContext::MD_unpredictable);
2935 if (!
DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2938 const Value *BOp0, *BOp1;
2941 Opcode = Instruction::And;
2943 Opcode = Instruction::Or;
2950 DAG.getTargetLoweringInfo().getJumpConditionMergingParams(
2951 Opcode, BOp0, BOp1,
FuncInfo.Fn))) {
2953 getEdgeProbability(BrMBB, Succ0MBB),
2954 getEdgeProbability(BrMBB, Succ1MBB),
2959 assert(
SL->SwitchCases[0].ThisBB == BrMBB &&
"Unexpected lowering!");
2963 for (
unsigned i = 1, e =
SL->SwitchCases.size(); i != e; ++i) {
2970 SL->SwitchCases.erase(
SL->SwitchCases.begin());
2976 for (
unsigned i = 1, e =
SL->SwitchCases.size(); i != e; ++i)
2977 FuncInfo.MF->erase(
SL->SwitchCases[i].ThisBB);
2979 SL->SwitchCases.clear();
2985 nullptr, Succ0MBB, Succ1MBB, BrMBB,
getCurSDLoc(),
3006 if (CB.
TrueBB != NextBlock(SwitchBB)) {
3013 auto &
TLI =
DAG.getTargetLoweringInfo();
3037 Cond =
DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.
CC);
3049 Cond =
DAG.getSetCC(dl, MVT::i1, CmpOp,
DAG.getConstant(
High, dl, VT),
3053 VT, CmpOp,
DAG.getConstant(
Low, dl, VT));
3054 Cond =
DAG.getSetCC(dl, MVT::i1, SUB,
3069 if (CB.
TrueBB == NextBlock(SwitchBB)) {
3085 BrCond =
DAG.getNode(
ISD::BR, dl, MVT::Other, BrCond,
3088 DAG.setRoot(BrCond);
3094 assert(JT.
SL &&
"Should set SDLoc for SelectionDAG!");
3095 assert(JT.
Reg &&
"Should lower JT Header first!");
3096 EVT PTy =
DAG.getTargetLoweringInfo().getJumpTableRegTy(
DAG.getDataLayout());
3100 Index.getValue(1),
Table, Index);
3101 DAG.setRoot(BrJumpTable);
3109 assert(JT.
SL &&
"Should set SDLoc for SelectionDAG!");
3116 DAG.getConstant(JTH.
First, dl, VT));
3125 DAG.getZExtOrTrunc(
Sub, dl,
TLI.getJumpTableRegTy(
DAG.getDataLayout()));
3131 JT.
Reg = JumpTableReg;
3138 dl,
TLI.getSetCCResultType(
DAG.getDataLayout(), *
DAG.getContext(),
3139 Sub.getValueType()),
3143 MVT::Other, CopyTo, CMP,
3147 if (JT.
MBB != NextBlock(SwitchBB))
3148 BrCond =
DAG.getNode(
ISD::BR, dl, MVT::Other, BrCond,
3149 DAG.getBasicBlock(JT.
MBB));
3151 DAG.setRoot(BrCond);
3154 if (JT.
MBB != NextBlock(SwitchBB))
3156 DAG.getBasicBlock(JT.
MBB)));
3158 DAG.setRoot(CopyTo);
3182 if (PtrTy != PtrMemTy)
3198 auto &
DL =
DAG.getDataLayout();
3199 EVT PtrTy =
TLI.getFrameIndexTy(
DL);
3200 EVT PtrMemTy =
TLI.getPointerMemTy(
DL,
DL.getAllocaAddrSpace());
3207 SDValue StackSlotPtr =
DAG.getFrameIndex(FI, PtrTy);
3214 PtrMemTy, dl,
DAG.getEntryNode(), StackSlotPtr,
3222 if (
TLI.useStackGuardMixFP())
3223 GuardVal =
TLI.emitStackGuardMixFP(
DAG, GuardVal, dl);
3231 TLI.getSSPStackGuardCheck(M,
DAG.getLibcalls());
3232 assert(GuardCheckFn &&
"Guard check function is null");
3243 Entry.IsInReg =
true;
3244 Args.push_back(Entry);
3250 getValue(GuardCheckFn), std::move(Args));
3252 std::pair<SDValue, SDValue> Result =
TLI.LowerCallTo(CLI);
3253 DAG.setRoot(Result.second);
3261 if (
TLI.useStackGuardMixFP()) {
3263 if (
const Value *IRGuard =
TLI.getSDagStackGuard(M,
DAG.getLibcalls())) {
3265 Guard =
DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3271 Guard =
DAG.getPOISON(PtrMemTy);
3275 if (
TLI.useLoadStackGuardNode(M)) {
3278 if (
const Value *IRGuard =
TLI.getSDagStackGuard(M,
DAG.getLibcalls())) {
3280 Guard =
DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3286 Guard =
DAG.getPOISON(PtrMemTy);
3329 auto *GuardCheckFn =
TLI.getSSPStackGuardCheck(M,
DAG.getLibcalls());
3332 auto &
DL =
DAG.getDataLayout();
3333 EVT PtrTy =
TLI.getFrameIndexTy(
DL);
3334 EVT PtrMemTy =
TLI.getPointerMemTy(
DL,
DL.getAllocaAddrSpace());
3340 SDValue StackSlotPtr =
DAG.getFrameIndex(FI, PtrTy);
3346 PtrMemTy, dl,
DAG.getEntryNode(), StackSlotPtr,
3350 if (
TLI.useStackGuardMixFP())
3351 GuardVal =
TLI.emitStackGuardMixFP(
DAG, GuardVal, dl);
3361 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3362 Entry.IsInReg =
true;
3363 Args.push_back(Entry);
3369 getValue(GuardCheckFn), std::move(Args));
3371 Chain =
TLI.LowerCallTo(CLI).second;
3375 Chain =
TLI.makeLibCall(
DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
3398 DAG.getNode(
ISD::SUB, dl, VT, SwitchOp,
DAG.getConstant(
B.First, dl, VT));
3402 bool UsePtrType =
false;
3403 if (!
TLI.isTypeLegal(VT)) {
3416 VT =
TLI.getPointerTy(
DAG.getDataLayout());
3426 if (!
B.FallthroughUnreachable)
3427 addSuccessorWithProb(SwitchBB,
B.Default,
B.DefaultProb);
3428 addSuccessorWithProb(SwitchBB,
MBB,
B.Prob);
3432 if (!
B.FallthroughUnreachable) {
3435 TLI.getSetCCResultType(
DAG.getDataLayout(), *
DAG.getContext(),
3441 DAG.getBasicBlock(
B.Default));
3445 if (
MBB != NextBlock(SwitchBB))
3463 if (PopCount == 1) {
3467 dl,
TLI.getSetCCResultType(
DAG.getDataLayout(), *
DAG.getContext(), VT),
3470 }
else if (PopCount == BB.
Range) {
3473 dl,
TLI.getSetCCResultType(
DAG.getDataLayout(), *
DAG.getContext(), VT),
3478 DAG.getConstant(1, dl, VT), ShiftOp);
3482 VT, SwitchVal,
DAG.getConstant(
B.Mask, dl, VT));
3484 dl,
TLI.getSetCCResultType(
DAG.getDataLayout(), *
DAG.getContext(), VT),
3489 addSuccessorWithProb(SwitchBB,
B.TargetBB,
B.ExtraProb);
3491 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3499 Cmp,
DAG.getBasicBlock(
B.TargetBB));
3502 if (NextMBB != NextBlock(SwitchBB))
3503 BrAnd =
DAG.getNode(
ISD::BR, dl, MVT::Other, BrAnd,
3504 DAG.getBasicBlock(NextMBB));
3509void SelectionDAGBuilder::visitInvoke(
const InvokeInst &
I) {
3527 const Value *Callee(
I.getCalledOperand());
3530 visitInlineAsm(
I, EHPadBB);
3535 case Intrinsic::donothing:
3537 case Intrinsic::seh_try_begin:
3538 case Intrinsic::seh_scope_begin:
3539 case Intrinsic::seh_try_end:
3540 case Intrinsic::seh_scope_end:
3546 case Intrinsic::experimental_patchpoint_void:
3547 case Intrinsic::experimental_patchpoint:
3548 visitPatchpoint(
I, EHPadBB);
3550 case Intrinsic::experimental_gc_statepoint:
3556 case Intrinsic::wasm_throw: {
3558 std::array<SDValue, 4>
Ops = {
3569 case Intrinsic::wasm_rethrow: {
3570 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
3571 std::array<SDValue, 2>
Ops = {
3580 case Intrinsic::bpf_unwind_exit: {
3581 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
3583 std::array<SDValue, 2>
Ops = {
3589 DAG.setRoot(lowerEndEH(Chain, &
I, EHPadBB, BeginLabel));
3593 }
else if (
I.hasDeoptState()) {
3614 BranchProbabilityInfo *BPI =
FuncInfo.BPI;
3615 BranchProbability EHPadBBProb =
3621 addSuccessorWithProb(InvokeMBB, Return);
3622 for (
auto &UnwindDest : UnwindDests) {
3623 UnwindDest.first->setIsEHPad();
3624 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3630 DAG.getBasicBlock(Return)));
3639void SelectionDAGBuilder::visitCallBrIntrinsic(
const CallBrInst &
I) {
3642 DAG.getTargetLoweringInfo().getTgtMemIntrinsic(
3643 Infos,
I,
DAG.getMachineFunction(),
I.getIntrinsicID());
3644 assert(Infos.
empty() &&
"Intrinsic touches memory");
3647 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(
I);
3650 getTargetIntrinsicOperands(
I, HasChain, OnlyLoad);
3651 SDVTList VTs = getTargetIntrinsicVTList(
I, HasChain);
3655 getTargetNonMemIntrinsicNode(*
I.getType(), HasChain,
Ops, VTs);
3656 Result = handleTargetIntrinsicRet(
I, HasChain, OnlyLoad, Result);
3661void SelectionDAGBuilder::visitCallBr(
const CallBrInst &
I) {
3662 MachineBasicBlock *CallBrMBB =
FuncInfo.MBB;
3664 if (
I.isInlineAsm()) {
3671 assert(!
I.hasOperandBundles() &&
3672 "Can't have operand bundles for intrinsics");
3673 visitCallBrIntrinsic(
I);
3678 SmallPtrSet<BasicBlock *, 8> Dests;
3679 Dests.
insert(
I.getDefaultDest());
3689 if (
I.isInlineAsm()) {
3690 for (BasicBlock *Dest :
I.getIndirectDests()) {
3692 Target->setIsInlineAsmBrIndirectTarget();
3698 Target->setLabelMustBeEmitted();
3700 if (Dests.
insert(Dest).second)
3709 DAG.getBasicBlock(Return)));
3712void SelectionDAGBuilder::visitResume(
const ResumeInst &RI) {
3713 llvm_unreachable(
"SelectionDAGBuilder shouldn't visit resume instructions!");
3716void SelectionDAGBuilder::visitLandingPad(
const LandingPadInst &LP) {
3718 "Call to landingpad not in landing pad!");
3722 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
3727 PersonalityFn) == 0)
3741 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
3743 "landingpad result type must be a struct of an exception pointer and "
3744 "an integer selector",
3752 assert(ValueVTs.
size() == 2 &&
"Only two-valued landingpads are supported");
3757 if (
FuncInfo.ExceptionPointerVirtReg) {
3758 Ops[0] =
DAG.getZExtOrTrunc(
3759 DAG.getCopyFromReg(
DAG.getEntryNode(), dl,
3766 Ops[1] =
DAG.getZExtOrTrunc(
3767 DAG.getCopyFromReg(
DAG.getEntryNode(), dl,
3774 DAG.getVTList(ValueVTs),
Ops);
3782 if (JTB.first.HeaderBB ==
First)
3783 JTB.first.HeaderBB =
Last;
3796 for (
unsigned i = 0, e =
I.getNumSuccessors(); i != e; ++i) {
3798 bool Inserted =
Done.insert(BB).second;
3803 addSuccessorWithProb(IndirectBrMBB, Succ);
3813 if (!
I.shouldLowerToTrap(
DAG.getTarget().Options.TrapUnreachable,
3814 DAG.getTarget().Options.NoTrapAfterNoreturn))
3820void SelectionDAGBuilder::visitUnary(
const User &
I,
unsigned Opcode) {
3823 Flags.copyFMF(*FPOp);
3831void SelectionDAGBuilder::visitBinary(
const User &
I,
unsigned Opcode) {
3834 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3835 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3838 Flags.setExact(ExactOp->isExact());
3840 Flags.setDisjoint(DisjointOp->isDisjoint());
3842 Flags.copyFMF(*FPOp);
3851void SelectionDAGBuilder::visitShift(
const User &
I,
unsigned Opcode) {
3855 EVT ShiftTy =
DAG.getTargetLoweringInfo().getShiftAmountTy(
3860 if (!
I.getType()->isVectorTy() && Op2.
getValueType() != ShiftTy) {
3862 "Unexpected shift type");
3872 if (
const OverflowingBinaryOperator *OFBinOp =
3874 nuw = OFBinOp->hasNoUnsignedWrap();
3875 nsw = OFBinOp->hasNoSignedWrap();
3877 if (
const PossiblyExactOperator *ExactOp =
3879 exact = ExactOp->isExact();
3882 Flags.setExact(exact);
3883 Flags.setNoSignedWrap(nsw);
3884 Flags.setNoUnsignedWrap(nuw);
3890void SelectionDAGBuilder::visitSDiv(
const User &
I) {
3901void SelectionDAGBuilder::visitICmp(
const ICmpInst &
I) {
3907 auto &
TLI =
DAG.getTargetLoweringInfo();
3920 Flags.setSameSign(
I.hasSameSign());
3922 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
3928void SelectionDAGBuilder::visitFCmp(
const FCmpInst &
I) {
3935 if (FPMO->hasNoNaNs() ||
3936 (
DAG.isKnownNeverNaN(Op1) &&
DAG.isKnownNeverNaN(Op2)))
3940 Flags.copyFMF(*FPMO);
3942 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
3952 return isa<SelectInst>(V);
3956void SelectionDAGBuilder::visitSelect(
const User &
I) {
3960 unsigned NumValues = ValueVTs.
size();
3961 if (NumValues == 0)
return;
3971 bool IsUnaryAbs =
false;
3972 bool Negate =
false;
3976 Flags.copyFMF(*FPOp);
3978 Flags.setUnpredictable(
3983 EVT VT = ValueVTs[0];
3984 LLVMContext &Ctx = *
DAG.getContext();
3985 auto &
TLI =
DAG.getTargetLoweringInfo();
3995 bool UseScalarMinMax = VT.
isVector() &&
4004 switch (SPR.Flavor) {
4013 switch (SPR.NaNBehavior) {
4020 Flags.setNoSignedZeros(
true);
4028 switch (SPR.NaNBehavior) {
4035 Flags.setNoSignedZeros(
true);
4071 for (
unsigned i = 0; i != NumValues; ++i) {
4080 for (
unsigned i = 0; i != NumValues; ++i) {
4094void SelectionDAGBuilder::visitTrunc(
const User &
I) {
4097 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4101 Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());
4102 Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());
4108void SelectionDAGBuilder::visitZExt(
const User &
I) {
4112 auto &
TLI =
DAG.getTargetLoweringInfo();
4117 Flags.setNonNeg(PNI->hasNonNeg());
4122 if (
Flags.hasNonNeg() &&
4131void SelectionDAGBuilder::visitSExt(
const User &
I) {
4135 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4140void SelectionDAGBuilder::visitFPTrunc(
const User &
I) {
4146 Flags.copyFMF(*FPOp);
4147 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4150 DAG.getTargetConstant(
4155void SelectionDAGBuilder::visitFPExt(
const User &
I) {
4158 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4162 Flags.copyFMF(*FPOp);
4166void SelectionDAGBuilder::visitFPToUI(
const User &
I) {
4169 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4174void SelectionDAGBuilder::visitFPToSI(
const User &
I) {
4177 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4182void SelectionDAGBuilder::visitUIToFP(
const User &
I) {
4185 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4194void SelectionDAGBuilder::visitSIToFP(
const User &
I) {
4197 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4205void SelectionDAGBuilder::visitPtrToAddr(
const User &
I) {
4208 const auto &
TLI =
DAG.getTargetLoweringInfo();
4216void SelectionDAGBuilder::visitPtrToInt(
const User &
I) {
4220 auto &
TLI =
DAG.getTargetLoweringInfo();
4221 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4230void SelectionDAGBuilder::visitIntToPtr(
const User &
I) {
4234 auto &
TLI =
DAG.getTargetLoweringInfo();
4242void SelectionDAGBuilder::visitBitCast(
const User &
I) {
4245 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4250 if (DestVT !=
N.getValueType())
4258 setValue(&
I,
DAG.getConstant(
C->getValue(), dl, DestVT,
false,
4264void SelectionDAGBuilder::visitAddrSpaceCast(
const User &
I) {
4265 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4266 const Value *
SV =
I.getOperand(0);
4270 unsigned SrcAS =
SV->getType()->getPointerAddressSpace();
4271 unsigned DestAS =
I.getType()->getPointerAddressSpace();
4273 if (!TM.isNoopAddrSpaceCast(
DAG.getDataLayout(), SrcAS, DestAS)) {
4276 Flags.setNonNull(ASC->hasNonNull());
4283void SelectionDAGBuilder::visitInsertElement(
const User &
I) {
4284 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4291 InVec, InVal, InIdx));
4294void SelectionDAGBuilder::visitBitInsert(
const User &
I) {
4298 EVT BaseVT =
Base.getValueType();
4300 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4304 "bitinsert val wider than base should be rejected by verifier");
4310 Val =
DAG.getBitcast(ValVT, Val);
4321 DAG.getNode(
ISD::SHL, dl, BaseVT,
DAG.getConstant(InsertMask, dl, BaseVT),
4323 SDValue ClearMask =
DAG.getNOT(dl, ShiftedMask, BaseVT);
4326 SDValue ExtVal =
DAG.getZExtOrTrunc(Val, dl, BaseVT);
4328 DAG.getNode(
ISD::SHL, dl, BaseVT, ExtVal, LegalShiftAmount);
4333void SelectionDAGBuilder::visitBitExtract(
const User &
I) {
4336 EVT SrcVT = Src.getValueType();
4337 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4342 "bitextract result wider than source should be rejected by verifier");
4356 Result =
DAG.getBitcast(ResultVT, Result);
4365void SelectionDAGBuilder::visitExtractElement(
const User &
I) {
4366 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4375void SelectionDAGBuilder::visitShuffleVector(
const User &
I) {
4380 Mask = SVI->getShuffleMask();
4384 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4386 EVT SrcVT =
Src1.getValueType();
4392 DAG.getVectorIdxConstant(0,
DL));
4403 unsigned MaskNumElts =
Mask.size();
4405 if (SrcNumElts == MaskNumElts) {
4411 if (SrcNumElts < MaskNumElts) {
4415 if (MaskNumElts % SrcNumElts == 0) {
4419 unsigned NumConcat = MaskNumElts / SrcNumElts;
4420 bool IsConcat =
true;
4421 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4422 for (
unsigned i = 0; i != MaskNumElts; ++i) {
4428 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4429 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4430 ConcatSrcs[i / SrcNumElts] != (
int)(Idx / SrcNumElts))) {
4435 ConcatSrcs[i / SrcNumElts] =
Idx / SrcNumElts;
4442 for (
auto Src : ConcatSrcs) {
4455 unsigned PaddedMaskNumElts =
alignTo(MaskNumElts, SrcNumElts);
4456 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4472 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4473 for (
unsigned i = 0; i != MaskNumElts; ++i) {
4475 if (Idx >= (
int)SrcNumElts)
4476 Idx -= SrcNumElts - PaddedMaskNumElts;
4484 if (MaskNumElts != PaddedMaskNumElts)
4486 DAG.getVectorIdxConstant(0,
DL));
4492 assert(SrcNumElts > MaskNumElts);
4496 int StartIdx[2] = {-1, -1};
4497 bool CanExtract =
true;
4498 for (
int Idx : Mask) {
4503 if (Idx >= (
int)SrcNumElts) {
4511 int NewStartIdx =
alignDown(Idx, MaskNumElts);
4512 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4513 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4517 StartIdx[
Input] = NewStartIdx;
4520 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4526 for (
unsigned Input = 0;
Input < 2; ++
Input) {
4528 if (StartIdx[Input] < 0)
4529 Src =
DAG.getUNDEF(VT);
4532 DAG.getVectorIdxConstant(StartIdx[Input],
DL));
4537 SmallVector<int, 8> MappedOps(Mask);
4538 for (
int &Idx : MappedOps) {
4539 if (Idx >= (
int)SrcNumElts)
4540 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4545 setValue(&
I,
DAG.getVectorShuffle(VT,
DL, Src1, Src2, MappedOps));
4554 for (
int Idx : Mask) {
4558 Res =
DAG.getUNDEF(EltVT);
4561 if (Idx >= (
int)SrcNumElts)
Idx -= SrcNumElts;
4564 DAG.getVectorIdxConstant(Idx,
DL));
4574 ArrayRef<unsigned> Indices =
I.getIndices();
4575 const Value *Op0 =
I.getOperand(0);
4577 Type *AggTy =
I.getType();
4584 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4590 unsigned NumAggValues = AggValueVTs.
size();
4591 unsigned NumValValues = ValValueVTs.
size();
4595 if (!NumAggValues) {
4603 for (; i != LinearIndex; ++i)
4604 Values[i] = IntoUndef ?
DAG.getUNDEF(AggValueVTs[i]) :
4609 for (; i != LinearIndex + NumValValues; ++i)
4610 Values[i] = FromUndef ?
DAG.getUNDEF(AggValueVTs[i]) :
4614 for (; i != NumAggValues; ++i)
4615 Values[i] = IntoUndef ?
DAG.getUNDEF(AggValueVTs[i]) :
4623 ArrayRef<unsigned> Indices =
I.getIndices();
4624 const Value *Op0 =
I.getOperand(0);
4626 Type *ValTy =
I.getType();
4631 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4635 unsigned NumValValues = ValValueVTs.
size();
4638 if (!NumValValues) {
4647 for (
unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4648 Values[i - LinearIndex] =
4657void SelectionDAGBuilder::visitGetElementPtr(
const User &
I) {
4658 Value *Op0 =
I.getOperand(0);
4664 auto &
TLI =
DAG.getTargetLoweringInfo();
4669 bool IsVectorGEP =
I.getType()->isVectorTy();
4670 ElementCount VectorElementCount =
4676 const Value *
Idx = GTI.getOperand();
4677 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4682 DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(
Field);
4692 N =
DAG.getMemBasePlusOffset(
4693 N,
DAG.getConstant(
Offset, dl,
N.getValueType()), dl, Flags);
4699 unsigned IdxSize =
DAG.getDataLayout().getIndexSizeInBits(AS);
4701 TypeSize ElementSize =
4702 GTI.getSequentialElementStride(
DAG.getDataLayout());
4707 bool ElementScalable = ElementSize.
isScalable();
4713 C =
C->getSplatValue();
4716 if (CI && CI->isZero())
4718 if (CI && !ElementScalable) {
4719 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4722 if (
N.getValueType().isVector())
4723 OffsVal =
DAG.getConstant(
4726 OffsVal =
DAG.getConstant(Offs, dl, IdxTy);
4733 Flags.setNoUnsignedWrap(
true);
4736 OffsVal =
DAG.getSExtOrTrunc(OffsVal, dl,
N.getValueType());
4738 N =
DAG.getMemBasePlusOffset(
N, OffsVal, dl, Flags);
4746 if (
N.getValueType().isVector()) {
4748 VectorElementCount);
4749 IdxN =
DAG.getSplat(VT, dl, IdxN);
4753 N =
DAG.getSplat(VT, dl,
N);
4759 IdxN =
DAG.getSExtOrTrunc(IdxN, dl,
N.getValueType());
4761 SDNodeFlags ScaleFlags;
4770 if (ElementScalable) {
4771 EVT VScaleTy =
N.getValueType().getScalarType();
4774 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4775 if (
N.getValueType().isVector())
4776 VScale =
DAG.getSplatVector(
N.getValueType(), dl, VScale);
4777 IdxN =
DAG.getNode(
ISD::MUL, dl,
N.getValueType(), IdxN, VScale,
4782 if (ElementMul != 1) {
4783 if (ElementMul.isPowerOf2()) {
4784 unsigned Amt = ElementMul.logBase2();
4787 DAG.getShiftAmountConstant(Amt,
N.getValueType(), dl),
4790 SDValue Scale =
DAG.getConstant(ElementMul.getZExtValue(), dl,
4792 IdxN =
DAG.getNode(
ISD::MUL, dl,
N.getValueType(), IdxN, Scale,
4802 SDNodeFlags AddFlags;
4806 N =
DAG.getMemBasePlusOffset(
N, IdxN, dl, AddFlags);
4810 if (IsVectorGEP && !
N.getValueType().isVector()) {
4812 N =
DAG.getSplat(VT, dl,
N);
4823 N =
DAG.getPtrExtendInReg(
N, dl, PtrMemTy);
4828void SelectionDAGBuilder::visitAlloca(
const AllocaInst &
I) {
4835 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4836 auto &
DL =
DAG.getDataLayout();
4837 TypeSize TySize =
I.getAllocationBaseSize(
DL);
4844 AllocSize =
DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4846 AllocSize =
DAG.getNode(
4848 DAG.getZExtOrTrunc(
DAG.getTypeSize(dl, MVT::i64, TySize), dl, IntPtr));
4853 Align StackAlign =
DAG.getSubtarget().getFrameLowering()->getStackAlign();
4854 if (*Alignment <= StackAlign)
4862 DAG.getConstant(StackAlignMask, dl, IntPtr),
4867 DAG.getSignedConstant(~StackAlignMask, dl, IntPtr));
4871 DAG.getConstant(Alignment ?
Alignment->value() : 0, dl, IntPtr)};
4881 return I.getMetadata(LLVMContext::MD_range);
4886 if (std::optional<ConstantRange> CR = CB->getRange())
4890 return std::nullopt;
4895 return CB->getRetNoFPClass();
4899void SelectionDAGBuilder::visitLoad(
const LoadInst &
I) {
4901 return visitAtomicLoad(
I);
4903 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4904 const Value *
SV =
I.getOperand(0);
4909 if (Arg->hasSwiftErrorAttr())
4910 return visitLoadFromSwiftError(
I);
4914 if (Alloca->isSwiftError())
4915 return visitLoadFromSwiftError(
I);
4921 Type *Ty =
I.getType();
4925 unsigned NumValues = ValueVTs.
size();
4930 AAMDNodes AAInfo =
I.getAAMetadata();
4933 bool isVolatile =
I.isVolatile();
4938 bool ConstantMemory =
false;
4945 BatchAA->pointsToConstantMemory(MemoryLocation(
4950 Root =
DAG.getEntryNode();
4951 ConstantMemory =
true;
4955 Root =
DAG.getRoot();
4966 unsigned ChainI = 0;
4967 for (
unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4983 MachinePointerInfo PtrInfo =
4985 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4986 : MachinePointerInfo();
4988 SDValue A =
DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4990 DAG.getLoad(MemVTs[i], dl, Root,
A, PtrInfo, Alignment, MMOFlags,
4991 MMOMetadata(AAInfo, Ranges, MemCacheHint));
4992 Chains[ChainI] =
L.getValue(1);
4994 if (MemVTs[i] != ValueVTs[i])
4995 L =
DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);
4997 if (MDNode *NoFPClassMD =
I.getMetadata(LLVMContext::MD_nofpclass)) {
4998 uint64_t FPTestInt =
5000 cast<ConstantAsMetadata>(NoFPClassMD->getOperand(0))->getValue())
5002 if (FPTestInt != fcNone) {
5003 SDValue FPTestConst =
5004 DAG.getTargetConstant(FPTestInt, SDLoc(), MVT::i32);
5005 L = DAG.getNode(ISD::AssertNoFPClass, dl, L.getValueType(), L,
5012 if (!ConstantMemory) {
5018 PendingLoads.push_back(Chain);
5022 DAG.getVTList(ValueVTs),
Values));
5025void SelectionDAGBuilder::visitStoreToSwiftError(
const StoreInst &
I) {
5026 assert(
DAG.getTargetLoweringInfo().supportSwiftError() &&
5027 "call visitStoreToSwiftError when backend supports swifterror");
5030 SmallVector<uint64_t, 4>
Offsets;
5031 const Value *SrcV =
I.getOperand(0);
5033 SrcV->
getType(), ValueVTs,
nullptr, &Offsets, 0);
5034 assert(ValueVTs.
size() == 1 && Offsets[0] == 0 &&
5035 "expect a single EVT for swifterror");
5044 SDValue(Src.getNode(), Src.getResNo()));
5045 DAG.setRoot(CopyNode);
5048void SelectionDAGBuilder::visitLoadFromSwiftError(
const LoadInst &
I) {
5049 assert(
DAG.getTargetLoweringInfo().supportSwiftError() &&
5050 "call visitLoadFromSwiftError when backend supports swifterror");
5053 !
I.hasMetadata(LLVMContext::MD_nontemporal) &&
5054 !
I.hasMetadata(LLVMContext::MD_invariant_load) &&
5055 "Support volatile, non temporal, invariant for load_from_swift_error");
5057 const Value *
SV =
I.getOperand(0);
5058 Type *Ty =
I.getType();
5061 !
BatchAA->pointsToConstantMemory(MemoryLocation(
5063 I.getAAMetadata()))) &&
5064 "load_from_swift_error should not be constant memory");
5067 SmallVector<uint64_t, 4>
Offsets;
5069 ValueVTs,
nullptr, &Offsets, 0);
5070 assert(ValueVTs.
size() == 1 && Offsets[0] == 0 &&
5071 "expect a single EVT for swifterror");
5081void SelectionDAGBuilder::visitStore(
const StoreInst &
I) {
5083 return visitAtomicStore(
I);
5085 const Value *SrcV =
I.getOperand(0);
5086 const Value *PtrV =
I.getOperand(1);
5088 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5093 if (Arg->hasSwiftErrorAttr())
5094 return visitStoreToSwiftError(
I);
5098 if (Alloca->isSwiftError())
5099 return visitStoreToSwiftError(
I);
5106 SrcV->
getType(), ValueVTs, &MemVTs, &Offsets);
5107 unsigned NumValues = ValueVTs.
size();
5121 AAMDNodes AAInfo =
I.getAAMetadata();
5126 unsigned ChainI = 0;
5127 for (
unsigned i = 0; i != NumValues; ++i, ++ChainI) {
5137 MachinePointerInfo PtrInfo =
5139 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
5140 : MachinePointerInfo();
5144 if (MemVTs[i] != ValueVTs[i])
5145 Val =
DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
5147 DAG.getStore(Root, dl, Val,
Add, PtrInfo, Alignment, MMOFlags,
5148 MMOMetadata(AAInfo,
nullptr, MemCacheHint));
5149 Chains[ChainI] = St;
5155 DAG.setRoot(StoreNode);
5158void SelectionDAGBuilder::visitMaskedStore(
const CallInst &
I,
5159 bool IsCompressing) {
5162 Value *Src0Operand =
I.getArgOperand(0);
5163 Value *PtrOperand =
I.getArgOperand(1);
5164 Value *MaskOperand =
I.getArgOperand(2);
5172 EVT VT =
Src0.getValueType();
5174 const auto &
TLI =
DAG.getTargetLoweringInfo();
5178 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
5183 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
5184 MachinePointerInfo(PtrOperand), MMOFlags,
5186 MMOMetadata(
I.getAAMetadata(),
nullptr, MemCacheHint));
5189 !IsCompressing &&
TTI->hasConditionalLoadStoreForType(
5190 I.getArgOperand(0)->getType(),
true)
5196 DAG.setRoot(StoreNode);
5226 C =
C->getSplatValue();
5240 if (!
GEP ||
GEP->getParent() != CurBB)
5243 if (
GEP->getNumOperands() != 2)
5246 const Value *BasePtr =
GEP->getPointerOperand();
5247 const Value *IndexVal =
GEP->getOperand(
GEP->getNumOperands() - 1);
5253 TypeSize ScaleVal =
DL.getTypeAllocSize(
GEP->getResultElementType());
5258 if (ScaleVal != 1 &&
5270void SelectionDAGBuilder::visitMaskedScatter(
const CallInst &
I) {
5274 const Value *Ptr =
I.getArgOperand(1);
5277 EVT VT =
Src0.getValueType();
5279 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5288 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
5298 EVT IdxVT =
Index.getValueType();
5306 SDValue Scatter =
DAG.getMaskedScatter(
DAG.getVTList(MVT::Other), VT, sdl,
5308 DAG.setRoot(Scatter);
5312void SelectionDAGBuilder::visitMaskedLoad(
const CallInst &
I,
bool IsExpanding) {
5315 Value *PtrOperand =
I.getArgOperand(0);
5316 Value *MaskOperand =
I.getArgOperand(1);
5317 Value *Src0Operand =
I.getArgOperand(2);
5325 EVT VT =
Src0.getValueType();
5326 AAMDNodes AAInfo =
I.getAAMetadata();
5334 SDValue InChain = AddToChain ?
DAG.getRoot() :
DAG.getEntryNode();
5336 const auto &
TLI =
DAG.getTargetLoweringInfo();
5340 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
5342 if (
I.hasMetadata(LLVMContext::MD_invariant_load))
5345 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
5346 MachinePointerInfo(PtrOperand), MMOFlags,
5348 MMOMetadata(AAInfo, Ranges, MemCacheHint));
5355 TTI->hasConditionalLoadStoreForType(Src0Operand->
getType(),
5360 DAG.getMaskedLoad(VT, sdl, InChain, Ptr,
Offset, Mask, Src0, VT, MMO,
5367void SelectionDAGBuilder::visitSpeculativeLoad(
const CallInst &
I) {
5369 Value *PtrOperand =
I.getArgOperand(0);
5374 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5377 AAMDNodes AAInfo =
I.getAAMetadata();
5385 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
5387 if (
I.hasMetadata(LLVMContext::MD_invariant_load))
5390 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
5391 MachinePointerInfo(PtrOperand), MMOFlags,
5399void SelectionDAGBuilder::visitMaskedGather(
const CallInst &
I) {
5403 const Value *Ptr =
I.getArgOperand(0);
5407 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5420 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
5423 MMOMetadata(
I.getAAMetadata(), Ranges));
5432 EVT IdxVT =
Index.getValueType();
5441 DAG.getMaskedGather(
DAG.getVTList(VT, MVT::Other), VT, sdl,
Ops, MMO,
5457 SDVTList VTs =
DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5459 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5465 MachinePointerInfo(
I.getPointerOperand()), Flags, MemVT.
getStoreSize(),
5466 I.getAlign(), MMOMetadata(AAMDNodes(),
nullptr, MemCacheHint),
5467 SSID, SuccessOrdering, FailureOrdering);
5470 dl, MemVT, VTs, InChain,
5478 DAG.setRoot(OutChain);
5481void SelectionDAGBuilder::visitAtomicRMW(
const AtomicRMWInst &
I) {
5484 switch (
I.getOperation()) {
5532 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5538 MachinePointerInfo(
I.getPointerOperand()), Flags, MemVT.
getStoreSize(),
5539 I.getAlign(), MMOMetadata(AAMDNodes(),
nullptr, MemCacheHint),
5543 DAG.getAtomic(NT, dl, MemVT, InChain,
5550 DAG.setRoot(OutChain);
5553void SelectionDAGBuilder::visitFence(
const FenceInst &
I) {
5555 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5558 Ops[1] =
DAG.getTargetConstant((
unsigned)
I.getOrdering(), dl,
5560 Ops[2] =
DAG.getTargetConstant(
I.getSyncScopeID(), dl,
5567void SelectionDAGBuilder::visitAtomicLoad(
const LoadInst &
I) {
5574 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5585 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
5586 MachinePointerInfo(
I.getPointerOperand()), Flags, MemVT.
getStoreSize(),
5587 I.getAlign(), MMOMetadata(AAMDNodes(), Ranges, MemCacheHint), SSID,
5598 L =
DAG.getPtrExtOrTrunc(L, dl, VT);
5601 DAG.setRoot(OutChain);
5604void SelectionDAGBuilder::visitAtomicStore(
const StoreInst &
I) {
5612 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5624 MachinePointerInfo(
I.getPointerOperand()), Flags, MemVT.
getStoreSize(),
5625 I.getAlign(), MMOMetadata(AAMDNodes(),
nullptr, MemCacheHint),
5630 Val =
DAG.getPtrExtOrTrunc(Val, dl, MemVT);
5637 DAG.setRoot(OutChain);
5645std::pair<bool, bool>
5646SelectionDAGBuilder::getTargetIntrinsicCallProperties(
const CallBase &
I) {
5648 bool HasChain = !
F->doesNotAccessMemory();
5650 HasChain &&
F->onlyReadsMemory() &&
F->willReturn() &&
F->doesNotThrow();
5652 return {HasChain, OnlyLoad};
5656 const CallBase &
I,
bool HasChain,
bool OnlyLoad,
5658 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5665 Ops.push_back(
DAG.getRoot());
5678 for (
unsigned i = 0, e =
I.arg_size(); i != e; ++i) {
5679 const Value *Arg =
I.getArgOperand(i);
5680 if (!
I.paramHasAttr(i, Attribute::ImmArg)) {
5688 assert(CI->getBitWidth() <= 64 &&
5689 "large intrinsic immediates not handled");
5690 Ops.push_back(
DAG.getTargetConstant(*CI, SDLoc(), VT));
5697 if (std::optional<OperandBundleUse> Bundle =
5699 auto *Sym = Bundle->Inputs[0].get();
5702 Ops.push_back(SDSym);
5705 if (std::optional<OperandBundleUse> Bundle =
5707 Value *Token = Bundle->Inputs[0].get();
5709 assert(
Ops.back().getValueType() != MVT::Glue &&
5710 "Did not expect another glue node here.");
5713 Ops.push_back(ConvControlToken);
5721 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5729 return DAG.getVTList(ValueVTs);
5733SDValue SelectionDAGBuilder::getTargetNonMemIntrinsicNode(
5756 if (
I.getType()->isVoidTy())
5771void SelectionDAGBuilder::visitTargetIntrinsic(
const CallInst &
I,
5773 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(
I);
5776 if (!
DAG.getMachineFunction().getSubtarget().isIntrinsicSupported(
5779 DAG.getContext()->diagnose(DiagnosticInfoUnsupportedTargetIntrinsic(
5780 *
I.getFunction(), IntrinsicID,
DL.getDebugLoc()));
5785 if (HasChain && !OnlyLoad)
5794 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5797 TargetLowering::IntrinsicInfo *
Info = !Infos.
empty() ? &Infos[0] :
nullptr;
5800 getTargetIntrinsicOperands(
I, HasChain, OnlyLoad, Info);
5801 SDVTList VTs = getTargetIntrinsicVTList(
I, HasChain);
5806 Flags.copyFMF(*FPMO);
5807 SelectionDAG::FlagInserter FlagsInserter(
DAG, Flags);
5814 if (!Infos.
empty()) {
5819 for (
const auto &Info : Infos) {
5822 MachinePointerInfo MPI;
5824 MPI = MachinePointerInfo(
Info.ptrVal,
Info.offset);
5825 else if (
Info.fallbackAddressSpace)
5826 MPI = MachinePointerInfo(*
Info.fallbackAddressSpace);
5827 EVT MemVT =
Info.memVT;
5829 if (
Size.hasValue() && !
Size.getValue())
5833 MPI,
Info.flags,
Size, Alignment,
I.getAAMetadata(),
Info.ssid,
5841 Result = getTargetNonMemIntrinsicNode(*
I.getType(), HasChain,
Ops, VTs);
5844 Result = handleTargetIntrinsicRet(
I, HasChain, OnlyLoad, Result);
5901 SDValue TwoToFractionalPartOfX;
5978 if (
Op.getValueType() == MVT::f32 &&
6002 if (
Op.getValueType() == MVT::f32 &&
6101 if (
Op.getValueType() == MVT::f32 &&
6185 return DAG.
getNode(
ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
6198 if (
Op.getValueType() == MVT::f32 &&
6275 return DAG.
getNode(
ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
6286 if (
Op.getValueType() == MVT::f32 &&
6299 bool IsExp10 =
false;
6300 if (
LHS.getValueType() == MVT::f32 &&
RHS.getValueType() == MVT::f32 &&
6304 IsExp10 = LHSC->isExactlyValue(Ten);
6331 unsigned Val = RHSC->getSExtValue();
6360 CurSquare, CurSquare);
6365 if (RHSC->getSExtValue() < 0)
6379 EVT VT =
LHS.getValueType();
6402 if ((ScaleInt > 0 || (Saturating &&
Signed)) &&
6403 (
TLI.isTypeLegal(VT) ||
6406 Opcode, VT, ScaleInt);
6441 switch (
N.getOpcode()) {
6445 Op.getValueType().getSizeInBits());
6470bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
6478 const TargetInstrInfo *
TII =
DAG.getSubtarget().getInstrInfo();
6482 auto MakeVRegDbgValue = [&](
Register Reg, DIExpression *FragExpr,
6487 auto &Inst =
TII->get(TargetOpcode::DBG_INSTR_REF);
6494 auto *NewDIExpr = FragExpr;
6501 return BuildMI(MF,
DL, Inst,
false, MOs, Variable, NewDIExpr);
6504 auto &Inst =
TII->get(TargetOpcode::DBG_VALUE);
6505 return BuildMI(MF,
DL, Inst, Indirect,
Reg, Variable, FragExpr);
6509 if (Kind == FuncArgumentDbgValueKind::Value) {
6514 if (!IsInEntryBlock)
6530 bool VariableIsFunctionInputArg =
Variable->isParameter() &&
6531 !
DL->getInlinedAt();
6533 if (!IsInPrologue && !VariableIsFunctionInputArg)
6567 if (VariableIsFunctionInputArg) {
6569 if (ArgNo >=
FuncInfo.DescribedArgs.size())
6570 FuncInfo.DescribedArgs.resize(ArgNo + 1,
false);
6571 else if (!IsInPrologue &&
FuncInfo.DescribedArgs.test(ArgNo))
6572 return !NodeMap[
V].getNode();
6577 bool IsIndirect =
false;
6578 std::optional<MachineOperand>
Op;
6580 int FI =
FuncInfo.getArgumentFrameIndex(Arg);
6581 if (FI != std::numeric_limits<int>::max())
6585 if (!
Op &&
N.getNode()) {
6588 if (ArgRegsAndSizes.
size() == 1)
6589 Reg = ArgRegsAndSizes.
front().first;
6592 MachineRegisterInfo &RegInfo = MF.
getRegInfo();
6599 IsIndirect =
Kind != FuncArgumentDbgValueKind::Value;
6603 if (!
Op &&
N.getNode()) {
6607 if (FrameIndexSDNode *FINode =
6614 auto splitMultiRegDbgValue =
6627 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6630 if (
Offset >= ExprFragmentSizeInBits)
6634 if (
Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6635 RegFragmentSizeInBits = ExprFragmentSizeInBits -
Offset;
6640 Expr,
Offset, RegFragmentSizeInBits);
6644 if (!FragmentExpr) {
6645 SDDbgValue *SDV =
DAG.getConstantDbgValue(
6647 DAG.AddDbgValue(SDV,
false);
6650 MachineInstr *NewMI = MakeVRegDbgValue(
6651 Reg, *FragmentExpr, Kind != FuncArgumentDbgValueKind::Value);
6652 FuncInfo.ArgDbgValues.push_back(NewMI);
6661 if (VMI !=
FuncInfo.ValueMap.end()) {
6662 const auto &
TLI =
DAG.getTargetLoweringInfo();
6663 RegsForValue RFV(
V->getContext(),
TLI,
DAG.getDataLayout(), VMI->second,
6664 V->getType(), std::nullopt);
6665 if (RFV.occupiesMultipleRegs())
6666 return splitMultiRegDbgValue(RFV.getRegsAndSizes());
6669 IsIndirect =
Kind != FuncArgumentDbgValueKind::Value;
6670 }
else if (ArgRegsAndSizes.
size() > 1) {
6673 return splitMultiRegDbgValue(ArgRegsAndSizes);
6681 "Expected inlined-at fields to agree");
6682 MachineInstr *NewMI =
nullptr;
6685 NewMI = MakeVRegDbgValue(
Op->getReg(), Expr, IsIndirect);
6687 NewMI =
BuildMI(MF,
DL,
TII->get(TargetOpcode::DBG_VALUE),
true, *
Op,
6691 FuncInfo.ArgDbgValues.push_back(NewMI);
6700 unsigned DbgSDNodeOrder) {
6712 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
6713 false, dl, DbgSDNodeOrder);
6715 return DAG.getDbgValue(Variable, Expr,
N.getNode(),
N.getResNo(),
6716 false, dl, DbgSDNodeOrder);
6721 case Intrinsic::smul_fix:
6723 case Intrinsic::umul_fix:
6725 case Intrinsic::smul_fix_sat:
6727 case Intrinsic::umul_fix_sat:
6729 case Intrinsic::sdiv_fix:
6731 case Intrinsic::udiv_fix:
6733 case Intrinsic::sdiv_fix_sat:
6735 case Intrinsic::udiv_fix_sat:
6748 "expected call_preallocated_setup Value");
6749 for (
const auto *U : PreallocatedSetup->
users()) {
6751 const Function *Fn = UseCall->getCalledFunction();
6752 if (!Fn || Fn->
getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6762bool SelectionDAGBuilder::visitEntryValueDbgValue(
6772 auto ArgIt =
FuncInfo.ValueMap.find(Arg);
6773 if (ArgIt ==
FuncInfo.ValueMap.end()) {
6775 dbgs() <<
"Dropping dbg.value: expression is entry_value but "
6776 "couldn't find an associated register for the Argument\n");
6779 Register ArgVReg = ArgIt->getSecond();
6781 for (
auto [PhysReg, VirtReg] :
FuncInfo.RegInfo->liveins())
6782 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6783 SDDbgValue *SDV =
DAG.getVRegDbgValue(
6784 Variable, Expr, PhysReg,
false , DbgLoc, SDNodeOrder);
6785 DAG.AddDbgValue(SDV,
false );
6788 LLVM_DEBUG(
dbgs() <<
"Dropping dbg.value: expression is entry_value but "
6789 "couldn't find a physical register\n");
6794void SelectionDAGBuilder::visitConvergenceControl(
const CallInst &
I,
6797 switch (Intrinsic) {
6798 case Intrinsic::experimental_convergence_anchor:
6801 case Intrinsic::experimental_convergence_entry:
6804 case Intrinsic::experimental_convergence_loop: {
6806 auto *Token = Bundle->Inputs[0].get();
6814void SelectionDAGBuilder::visitVectorHistogram(
const CallInst &
I,
6815 unsigned IntrinsicID) {
6818 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6819 "Tried to lower unsupported histogram type");
6825 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
6840 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
6841 MachinePointerInfo(AS),
6844 MMOMetadata(
I.getAAMetadata(), Ranges));
6853 EVT IdxVT =
Index.getValueType();
6864 SDValue ID =
DAG.getTargetConstant(IntrinsicID, sdl, MVT::i32);
6867 SDValue Histogram =
DAG.getMaskedHistogram(
DAG.getVTList(MVT::Other), VT, sdl,
6871 DAG.setRoot(Histogram);
6874void SelectionDAGBuilder::visitVectorExtractLastActive(
const CallInst &
I,
6876 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6877 "Tried lowering invalid vector extract last");
6879 const DataLayout &Layout =
DAG.getDataLayout();
6883 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
6893 EVT BoolVT =
Mask.getValueType().getScalarType();
6895 Result =
DAG.getSelect(sdl, ResVT, AnyActive, Result, PassThru);
6902void SelectionDAGBuilder::visitIntrinsicCall(
const CallInst &
I,
6904 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
6911 Flags.copyFMF(*FPOp);
6913 switch (Intrinsic) {
6916 visitTargetIntrinsic(
I, Intrinsic);
6918 case Intrinsic::vscale: {
6923 case Intrinsic::vastart: visitVAStart(
I);
return;
6924 case Intrinsic::vaend: visitVAEnd(
I);
return;
6925 case Intrinsic::vacopy: visitVACopy(
I);
return;
6926 case Intrinsic::returnaddress:
6931 case Intrinsic::addressofreturnaddress:
6936 case Intrinsic::sponentry:
6941 case Intrinsic::frameaddress:
6946 case Intrinsic::read_volatile_register:
6947 case Intrinsic::read_register: {
6948 Value *
Reg =
I.getArgOperand(0);
6954 DAG.getVTList(VT, MVT::Other), Chain,
RegName);
6959 case Intrinsic::write_register: {
6960 Value *
Reg =
I.getArgOperand(0);
6961 Value *RegValue =
I.getArgOperand(1);
6969 case Intrinsic::write_volatile_register: {
6970 Value *
Reg =
I.getArgOperand(0);
6971 Value *RegValue =
I.getArgOperand(1);
6988 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
6989 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(PhysReg);
6990 MVT RegVT = *TRI->legalclasstypes_begin(*RC);
6991 DAG.setRoot(DAG.getNode(ISD::FAKE_USE, sdl, MVT::Other,
6992 {WriteChain, DAG.getRegister(PhysReg, RegVT)}));
6994 DAG.setRoot(WriteChain);
6998 case Intrinsic::memcpy:
6999 case Intrinsic::memcpy_inline: {
7005 "memcpy_inline needs constant size");
7007 Align DstAlign = MCI.getDestAlign().valueOrOne();
7008 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
7009 bool isVol = MCI.isVolatile();
7011 SDValue MC =
DAG.getMemcpy(Root, sdl, Dst, Src,
Size, DstAlign, SrcAlign,
7012 isVol, MCI.isForceInlined(), &
I, std::nullopt,
7013 MachinePointerInfo(
I.getArgOperand(0)),
7014 MachinePointerInfo(
I.getArgOperand(1)),
7016 updateDAGForMaybeTailCall(MC);
7019 case Intrinsic::memset:
7020 case Intrinsic::memset_inline: {
7026 "memset_inline needs constant size");
7028 Align DstAlign = MSII.getDestAlign().valueOrOne();
7029 bool isVol = MSII.isVolatile();
7032 Root, sdl, Dst, Value,
Size, DstAlign, isVol, MSII.isForceInlined(),
7033 &
I, MachinePointerInfo(
I.getArgOperand(0)),
I.getAAMetadata());
7034 updateDAGForMaybeTailCall(MC);
7037 case Intrinsic::memmove: {
7043 Align DstAlign = MMI.getDestAlign().valueOrOne();
7044 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
7045 bool isVol = MMI.isVolatile();
7048 Root, sdl, Op1, Op2, Op3, DstAlign, SrcAlign, isVol, &
I,
7050 MachinePointerInfo(
I.getArgOperand(0)),
7051 MachinePointerInfo(
I.getArgOperand(1)),
I.getAAMetadata(),
BatchAA);
7052 updateDAGForMaybeTailCall(MM);
7055 case Intrinsic::memcpy_element_unordered_atomic: {
7061 Type *LengthTy =
MI.getLength()->getType();
7062 unsigned ElemSz =
MI.getElementSizeInBytes();
7066 isTC, MachinePointerInfo(
MI.getRawDest()),
7067 MachinePointerInfo(
MI.getRawSource()));
7068 updateDAGForMaybeTailCall(MC);
7071 case Intrinsic::memmove_element_unordered_atomic: {
7077 Type *LengthTy =
MI.getLength()->getType();
7078 unsigned ElemSz =
MI.getElementSizeInBytes();
7082 isTC, MachinePointerInfo(
MI.getRawDest()),
7083 MachinePointerInfo(
MI.getRawSource()));
7084 updateDAGForMaybeTailCall(MC);
7087 case Intrinsic::memset_element_unordered_atomic: {
7093 Type *LengthTy =
MI.getLength()->getType();
7094 unsigned ElemSz =
MI.getElementSizeInBytes();
7098 isTC, MachinePointerInfo(
MI.getRawDest()));
7099 updateDAGForMaybeTailCall(MC);
7102 case Intrinsic::call_preallocated_setup: {
7104 SDValue SrcValue =
DAG.getSrcValue(PreallocatedCall);
7111 case Intrinsic::call_preallocated_arg: {
7113 SDValue SrcValue =
DAG.getSrcValue(PreallocatedCall);
7127 case Intrinsic::eh_typeid_for: {
7130 unsigned TypeID =
DAG.getMachineFunction().getTypeIDFor(GV);
7131 Res =
DAG.getConstant(
TypeID, sdl, MVT::i32);
7136 case Intrinsic::eh_return_i32:
7137 case Intrinsic::eh_return_i64:
7138 DAG.getMachineFunction().setCallsEHReturn(
true);
7145 case Intrinsic::eh_unwind_init:
7146 DAG.getMachineFunction().setCallsUnwindInit(
true);
7148 case Intrinsic::eh_dwarf_cfa:
7153 case Intrinsic::eh_sjlj_callsite: {
7155 assert(
FuncInfo.getCurrentCallSite() == 0 &&
"Overlapping call sites!");
7160 case Intrinsic::eh_sjlj_functioncontext: {
7162 MachineFrameInfo &MFI =
DAG.getMachineFunction().getFrameInfo();
7165 int FI =
FuncInfo.StaticAllocaMap[FnCtx];
7169 case Intrinsic::eh_sjlj_setjmp: {
7174 DAG.getVTList(MVT::i32, MVT::Other),
Ops);
7176 DAG.setRoot(
Op.getValue(1));
7179 case Intrinsic::eh_sjlj_longjmp:
7183 case Intrinsic::eh_sjlj_setup_dispatch:
7187 case Intrinsic::masked_gather:
7188 visitMaskedGather(
I);
7190 case Intrinsic::masked_load:
7193 case Intrinsic::masked_scatter:
7194 visitMaskedScatter(
I);
7196 case Intrinsic::masked_store:
7197 visitMaskedStore(
I);
7199 case Intrinsic::masked_expandload:
7200 visitMaskedLoad(
I,
true );
7202 case Intrinsic::masked_compressstore:
7203 visitMaskedStore(
I,
true );
7205 case Intrinsic::speculative_load:
7206 visitSpeculativeLoad(
I);
7208 case Intrinsic::powi:
7212 case Intrinsic::log:
7215 case Intrinsic::log2:
7219 case Intrinsic::log10:
7223 case Intrinsic::exp:
7226 case Intrinsic::exp2:
7230 case Intrinsic::pow:
7234 case Intrinsic::sqrt:
7235 case Intrinsic::fabs:
7236 case Intrinsic::sin:
7237 case Intrinsic::cos:
7238 case Intrinsic::tan:
7239 case Intrinsic::asin:
7240 case Intrinsic::acos:
7241 case Intrinsic::atan:
7242 case Intrinsic::sinh:
7243 case Intrinsic::cosh:
7244 case Intrinsic::tanh:
7245 case Intrinsic::exp10:
7246 case Intrinsic::floor:
7247 case Intrinsic::ceil:
7248 case Intrinsic::trunc:
7249 case Intrinsic::rint:
7250 case Intrinsic::nearbyint:
7251 case Intrinsic::round:
7252 case Intrinsic::roundeven:
7253 case Intrinsic::canonicalize: {
7256 switch (Intrinsic) {
7258 case Intrinsic::sqrt: Opcode =
ISD::FSQRT;
break;
7259 case Intrinsic::fabs: Opcode =
ISD::FABS;
break;
7260 case Intrinsic::sin: Opcode =
ISD::FSIN;
break;
7261 case Intrinsic::cos: Opcode =
ISD::FCOS;
break;
7262 case Intrinsic::tan: Opcode =
ISD::FTAN;
break;
7263 case Intrinsic::asin: Opcode =
ISD::FASIN;
break;
7264 case Intrinsic::acos: Opcode =
ISD::FACOS;
break;
7265 case Intrinsic::atan: Opcode =
ISD::FATAN;
break;
7266 case Intrinsic::sinh: Opcode =
ISD::FSINH;
break;
7267 case Intrinsic::cosh: Opcode =
ISD::FCOSH;
break;
7268 case Intrinsic::tanh: Opcode =
ISD::FTANH;
break;
7269 case Intrinsic::exp10: Opcode =
ISD::FEXP10;
break;
7270 case Intrinsic::floor: Opcode =
ISD::FFLOOR;
break;
7271 case Intrinsic::ceil: Opcode =
ISD::FCEIL;
break;
7272 case Intrinsic::trunc: Opcode =
ISD::FTRUNC;
break;
7273 case Intrinsic::rint: Opcode =
ISD::FRINT;
break;
7275 case Intrinsic::round: Opcode =
ISD::FROUND;
break;
7282 getValue(
I.getArgOperand(0)).getValueType(),
7286 case Intrinsic::atan2:
7288 getValue(
I.getArgOperand(0)).getValueType(),
7292 case Intrinsic::lround:
7293 case Intrinsic::llround:
7294 case Intrinsic::lrint:
7295 case Intrinsic::llrint: {
7298 switch (Intrinsic) {
7300 case Intrinsic::lround: Opcode =
ISD::LROUND;
break;
7302 case Intrinsic::lrint: Opcode =
ISD::LRINT;
break;
7303 case Intrinsic::llrint: Opcode =
ISD::LLRINT;
break;
7312 case Intrinsic::minnum:
7314 getValue(
I.getArgOperand(0)).getValueType(),
7318 case Intrinsic::maxnum:
7320 getValue(
I.getArgOperand(0)).getValueType(),
7324 case Intrinsic::minimum:
7326 getValue(
I.getArgOperand(0)).getValueType(),
7330 case Intrinsic::maximum:
7332 getValue(
I.getArgOperand(0)).getValueType(),
7336 case Intrinsic::minimumnum:
7338 getValue(
I.getArgOperand(0)).getValueType(),
7342 case Intrinsic::maximumnum:
7344 getValue(
I.getArgOperand(0)).getValueType(),
7348 case Intrinsic::copysign:
7350 getValue(
I.getArgOperand(0)).getValueType(),
7354 case Intrinsic::ldexp:
7356 getValue(
I.getArgOperand(0)).getValueType(),
7360 case Intrinsic::modf:
7361 case Intrinsic::sincos:
7362 case Intrinsic::sincospi:
7363 case Intrinsic::frexp: {
7365 switch (Intrinsic) {
7368 case Intrinsic::sincos:
7371 case Intrinsic::sincospi:
7374 case Intrinsic::modf:
7377 case Intrinsic::frexp:
7383 SDVTList VTs =
DAG.getVTList(ValueVTs);
7385 &
I,
DAG.getNode(Opcode, sdl, VTs,
getValue(
I.getArgOperand(0)), Flags));
7388 case Intrinsic::arithmetic_fence: {
7390 getValue(
I.getArgOperand(0)).getValueType(),
7394 case Intrinsic::fma:
7400#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
7401 case Intrinsic::INTRINSIC:
7402#include "llvm/IR/ConstrainedOps.def"
7405#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
7406#include "llvm/IR/VPIntrinsics.def"
7409 case Intrinsic::fptrunc_round: {
7413 std::optional<RoundingMode> RoundMode =
7421 SelectionDAG::FlagInserter FlagsInserter(
DAG, Flags);
7426 DAG.getTargetConstant((
int)*RoundMode, sdl, MVT::i32));
7431 case Intrinsic::fmuladd: {
7435 getValue(
I.getArgOperand(0)).getValueType(),
7442 getValue(
I.getArgOperand(0)).getValueType(),
7458 case Intrinsic::fptosi_sat: {
7465 case Intrinsic::fptoui_sat: {
7472 case Intrinsic::convert_from_arbitrary_fp: {
7477 const fltSemantics *SrcSem =
7480 DAG.getContext()->emitError(
7481 "convert_from_arbitrary_fp: not implemented format '" + FormatStr +
7492 DAG.getTargetConstant(
static_cast<int>(SemEnum), sdl, MVT::i32);
7497 case Intrinsic::convert_to_arbitrary_fp: {
7502 const fltSemantics *DstSem =
7505 DAG.getContext()->emitError(
7506 "convert_to_arbitrary_fp: not implemented format '" + FormatStr +
7518 "Dynamic rounding mode should have been rejected by the verifier");
7526 DAG.getTargetConstant(
static_cast<int>(SemEnum), sdl, MVT::i32);
7528 DAG.getTargetConstant(
static_cast<int>(*RoundMode), sdl, MVT::i32);
7529 SDValue SatConst =
DAG.getTargetConstant(Saturate, sdl, MVT::i32);
7531 SemConst, RoundConst, SatConst));
7534 case Intrinsic::set_rounding:
7540 case Intrinsic::is_fpclass: {
7541 const DataLayout DLayout =
DAG.getDataLayout();
7550 Flags.setNoFPExcept(
7551 !
F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
7567 case Intrinsic::get_fpenv: {
7568 const DataLayout DLayout =
DAG.getDataLayout();
7570 Align TempAlign =
DAG.getEVTAlign(EnvVT);
7585 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
7588 Chain =
DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7589 Res =
DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
7595 case Intrinsic::set_fpenv: {
7598 Align TempAlign =
DAG.getEVTAlign(EnvVT);
7611 Chain =
DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
7613 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
7616 Chain =
DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7621 case Intrinsic::reset_fpenv:
7624 case Intrinsic::get_fpmode:
7633 case Intrinsic::set_fpmode:
7638 case Intrinsic::reset_fpmode: {
7643 case Intrinsic::pcmarker: {
7648 case Intrinsic::readcyclecounter: {
7651 DAG.getVTList(MVT::i64, MVT::Other),
Op);
7656 case Intrinsic::readsteadycounter: {
7659 DAG.getVTList(MVT::i64, MVT::Other),
Op);
7664 case Intrinsic::bitreverse:
7666 getValue(
I.getArgOperand(0)).getValueType(),
7669 case Intrinsic::bswap:
7671 getValue(
I.getArgOperand(0)).getValueType(),
7674 case Intrinsic::cttz: {
7682 case Intrinsic::ctlz: {
7690 case Intrinsic::ctpop: {
7696 case Intrinsic::fshl:
7697 case Intrinsic::fshr: {
7698 bool IsFSHL =
Intrinsic == Intrinsic::fshl;
7702 EVT VT =
X.getValueType();
7713 case Intrinsic::clmul: {
7719 case Intrinsic::pext: {
7725 case Intrinsic::pdep: {
7731 case Intrinsic::smulh:
7732 case Intrinsic::umulh: {
7739 case Intrinsic::sadd_sat: {
7745 case Intrinsic::uadd_sat: {
7751 case Intrinsic::ssub_sat: {
7757 case Intrinsic::usub_sat: {
7763 case Intrinsic::sshl_sat:
7764 case Intrinsic::ushl_sat: {
7768 EVT ShiftTy =
DAG.getTargetLoweringInfo().getShiftAmountTy(
7773 if (!
I.getType()->isVectorTy() && Op2.
getValueType() != ShiftTy) {
7776 "Unexpected shift type");
7785 case Intrinsic::smul_fix:
7786 case Intrinsic::umul_fix:
7787 case Intrinsic::smul_fix_sat:
7788 case Intrinsic::umul_fix_sat: {
7796 case Intrinsic::sdiv_fix:
7797 case Intrinsic::udiv_fix:
7798 case Intrinsic::sdiv_fix_sat:
7799 case Intrinsic::udiv_fix_sat: {
7804 Op1, Op2, Op3,
DAG,
TLI));
7807 case Intrinsic::smax: {
7813 case Intrinsic::smin: {
7819 case Intrinsic::umax: {
7825 case Intrinsic::umin: {
7831 case Intrinsic::abs: {
7838 case Intrinsic::scmp: {
7845 case Intrinsic::ucmp: {
7852 case Intrinsic::stackaddress:
7853 case Intrinsic::stacksave: {
7858 Res =
DAG.getNode(SDOpcode, sdl,
DAG.getVTList(VT, MVT::Other),
Op);
7863 case Intrinsic::stackrestore:
7867 case Intrinsic::get_dynamic_area_offset: {
7876 case Intrinsic::stackguard: {
7883 Res =
DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);
7887 LLVMContext &Ctx = *
DAG.getContext();
7888 Ctx.
diagnose(DiagnosticInfoGeneric(
"unable to lower stackguard"));
7895 MachinePointerInfo(
Global, 0), Align,
7907 case Intrinsic::stackprotector: {
7929 Chain, sdl, Src, FIN,
7936 case Intrinsic::objectsize:
7939 case Intrinsic::is_constant:
7942 case Intrinsic::annotation:
7943 case Intrinsic::ptr_annotation:
7944 case Intrinsic::launder_invariant_group:
7949 case Intrinsic::type_test:
7950 case Intrinsic::public_type_test:
7951 case Intrinsic::type_checked_load:
7952 case Intrinsic::type_checked_load_relative: {
7957 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
7960 " intrinsic must be lowered by the LowerTypeTests pass "
7961 "before code generation",
7970 case Intrinsic::assume:
7971 case Intrinsic::experimental_noalias_scope_decl:
7972 case Intrinsic::var_annotation:
7973 case Intrinsic::sideeffect:
7978 case Intrinsic::codeview_annotation: {
7989 case Intrinsic::init_trampoline: {
7997 Ops[4] =
DAG.getSrcValue(
I.getArgOperand(0));
8005 case Intrinsic::adjust_trampoline:
8010 case Intrinsic::gcroot: {
8011 assert(
DAG.getMachineFunction().getFunction().hasGC() &&
8012 "only valid in functions with gc specified, enforced by Verifier");
8014 const Value *Alloca =
I.getArgOperand(0)->stripPointerCasts();
8021 case Intrinsic::gcread:
8022 case Intrinsic::gcwrite:
8024 case Intrinsic::get_rounding:
8030 case Intrinsic::expect:
8031 case Intrinsic::expect_with_probability:
8037 case Intrinsic::ubsantrap:
8038 case Intrinsic::debugtrap:
8039 case Intrinsic::trap: {
8040 StringRef TrapFuncName =
8041 I.getAttributes().getFnAttr(
"trap-func-name").getValueAsString();
8042 if (TrapFuncName.
empty()) {
8043 switch (Intrinsic) {
8044 case Intrinsic::trap:
8047 case Intrinsic::debugtrap:
8050 case Intrinsic::ubsantrap:
8053 DAG.getTargetConstant(
8059 DAG.addNoMergeSiteInfo(
DAG.getRoot().getNode(),
8060 I.hasFnAttr(Attribute::NoMerge));
8064 if (Intrinsic == Intrinsic::ubsantrap) {
8065 Value *Arg =
I.getArgOperand(0);
8067 LibInfo->getExtAttrForI8Param(
false));
8070 TargetLowering::CallLoweringInfo CLI(
DAG);
8071 CLI.setDebugLoc(sdl).setChain(
getRoot()).setLibCallee(
8073 DAG.getExternalSymbol(TrapFuncName.
data(),
8076 CLI.NoMerge =
I.hasFnAttr(Attribute::NoMerge);
8082 case Intrinsic::allow_runtime_check:
8083 case Intrinsic::allow_ubsan_check:
8087 case Intrinsic::uadd_with_overflow:
8088 case Intrinsic::sadd_with_overflow:
8089 case Intrinsic::usub_with_overflow:
8090 case Intrinsic::ssub_with_overflow:
8091 case Intrinsic::umul_with_overflow:
8092 case Intrinsic::smul_with_overflow: {
8094 switch (Intrinsic) {
8096 case Intrinsic::uadd_with_overflow:
Op =
ISD::UADDO;
break;
8097 case Intrinsic::sadd_with_overflow:
Op =
ISD::SADDO;
break;
8098 case Intrinsic::usub_with_overflow:
Op =
ISD::USUBO;
break;
8099 case Intrinsic::ssub_with_overflow:
Op =
ISD::SSUBO;
break;
8100 case Intrinsic::umul_with_overflow:
Op =
ISD::UMULO;
break;
8101 case Intrinsic::smul_with_overflow:
Op =
ISD::SMULO;
break;
8109 SDVTList VTs =
DAG.getVTList(ResultVT, OverflowVT);
8113 case Intrinsic::prefetch: {
8128 std::nullopt, Flags);
8134 DAG.setRoot(Result);
8137 case Intrinsic::lifetime_start:
8138 case Intrinsic::lifetime_end: {
8139 bool IsStart = (
Intrinsic == Intrinsic::lifetime_start);
8145 if (!LifetimeObject)
8150 auto SI =
FuncInfo.StaticAllocaMap.find(LifetimeObject);
8151 if (SI ==
FuncInfo.StaticAllocaMap.end())
8155 Res =
DAG.getLifetimeNode(IsStart, sdl,
getRoot(), FrameIndex);
8159 case Intrinsic::pseudoprobe: {
8167 case Intrinsic::invariant_start:
8172 case Intrinsic::invariant_end:
8175 case Intrinsic::clear_cache: {
8180 {InputChain, StartVal, EndVal});
8185 case Intrinsic::donothing:
8186 case Intrinsic::seh_try_begin:
8187 case Intrinsic::seh_scope_begin:
8188 case Intrinsic::seh_try_end:
8189 case Intrinsic::seh_scope_end:
8192 case Intrinsic::experimental_stackmap:
8195 case Intrinsic::experimental_patchpoint_void:
8196 case Intrinsic::experimental_patchpoint:
8199 case Intrinsic::experimental_gc_statepoint:
8202 case Intrinsic::experimental_gc_result:
8205 case Intrinsic::experimental_gc_relocate:
8208 case Intrinsic::instrprof_cover:
8210 case Intrinsic::instrprof_increment:
8212 case Intrinsic::instrprof_timestamp:
8214 case Intrinsic::instrprof_value_profile:
8216 case Intrinsic::instrprof_mcdc_parameters:
8218 case Intrinsic::instrprof_mcdc_tvbitmap_update:
8220 case Intrinsic::localescape: {
8222 const TargetInstrInfo *
TII =
DAG.getSubtarget().getInstrInfo();
8226 for (
unsigned Idx = 0,
E =
I.arg_size(); Idx <
E; ++Idx) {
8232 "can only escape static allocas");
8237 TII->get(TargetOpcode::LOCAL_ESCAPE))
8245 case Intrinsic::localrecover: {
8253 unsigned(
Idx->getLimitedValue(std::numeric_limits<int>::max()));
8257 Value *
FP =
I.getArgOperand(1);
8263 SDValue OffsetSym =
DAG.getMCSymbol(FrameAllocSym, PtrVT);
8268 SDValue Add =
DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
8274 case Intrinsic::fake_use: {
8275 Value *
V =
I.getArgOperand(0);
8280 auto FakeUseValue = [&]() ->
SDValue {
8294 if (!FakeUseValue || FakeUseValue.isUndef())
8297 Ops[1] = FakeUseValue;
8306 case Intrinsic::reloc_none: {
8311 DAG.getTargetExternalSymbol(
8317 case Intrinsic::cond_loop: {
8327 case Intrinsic::eh_exceptionpointer:
8328 case Intrinsic::eh_exceptioncode: {
8334 SDValue N =
DAG.getCopyFromReg(
DAG.getEntryNode(), sdl, VReg, PtrVT);
8335 if (Intrinsic == Intrinsic::eh_exceptioncode)
8336 N =
DAG.getZExtOrTrunc(
N, sdl, MVT::i32);
8340 case Intrinsic::xray_customevent: {
8343 const auto &Triple =
DAG.getTarget().getTargetTriple();
8344 if (!Triple.isAArch64(64) && Triple.getArch() !=
Triple::x86_64 &&
8353 SDVTList NodeTys =
DAG.getVTList(MVT::Other, MVT::Glue);
8355 Ops.push_back(LogEntryVal);
8356 Ops.push_back(StrSizeVal);
8357 Ops.push_back(Chain);
8363 MachineSDNode *MN =
DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
8366 DAG.setRoot(patchableNode);
8370 case Intrinsic::xray_typedevent: {
8373 const auto &Triple =
DAG.getTarget().getTargetTriple();
8374 if (!Triple.isAArch64(64) && Triple.getArch() !=
Triple::x86_64 &&
8386 SDVTList NodeTys =
DAG.getVTList(MVT::Other, MVT::Glue);
8388 Ops.push_back(LogTypeId);
8389 Ops.push_back(LogEntryVal);
8390 Ops.push_back(StrSizeVal);
8391 Ops.push_back(Chain);
8397 MachineSDNode *MN =
DAG.getMachineNode(
8398 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys,
Ops);
8400 DAG.setRoot(patchableNode);
8404 case Intrinsic::experimental_deoptimize:
8407 case Intrinsic::stepvector:
8410 case Intrinsic::vector_reduce_fadd:
8411 case Intrinsic::vector_reduce_fmul:
8412 case Intrinsic::vector_reduce_add:
8413 case Intrinsic::vector_reduce_mul:
8414 case Intrinsic::vector_reduce_and:
8415 case Intrinsic::vector_reduce_or:
8416 case Intrinsic::vector_reduce_xor:
8417 case Intrinsic::vector_reduce_smax:
8418 case Intrinsic::vector_reduce_smin:
8419 case Intrinsic::vector_reduce_umax:
8420 case Intrinsic::vector_reduce_umin:
8421 case Intrinsic::vector_reduce_fmax:
8422 case Intrinsic::vector_reduce_fmin:
8423 case Intrinsic::vector_reduce_fmaximum:
8424 case Intrinsic::vector_reduce_fminimum:
8425 case Intrinsic::vector_reduce_fmaximumnum:
8426 case Intrinsic::vector_reduce_fminimumnum:
8427 visitVectorReduce(
I, Intrinsic);
8430 case Intrinsic::icall_branch_funnel: {
8436 I.getArgOperand(1),
Offset,
DAG.getDataLayout()));
8439 "llvm.icall.branch.funnel operand must be a GlobalValue");
8440 Ops.push_back(
DAG.getTargetGlobalAddress(
Base, sdl, MVT::i64, 0));
8442 struct BranchFunnelTarget {
8448 for (
unsigned Op = 1,
N =
I.arg_size();
Op !=
N;
Op += 2) {
8451 if (ElemBase !=
Base)
8453 "to the same GlobalValue");
8459 "llvm.icall.branch.funnel operand must be a GlobalValue");
8465 [](
const BranchFunnelTarget &
T1,
const BranchFunnelTarget &T2) {
8466 return T1.Offset < T2.Offset;
8469 for (
auto &
T : Targets) {
8470 Ops.push_back(
DAG.getTargetConstant(
T.Offset, sdl, MVT::i32));
8471 Ops.push_back(
T.Target);
8474 Ops.push_back(
DAG.getRoot());
8475 SDValue N(
DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
8484 case Intrinsic::wasm_landingpad_index:
8490 case Intrinsic::aarch64_settag:
8491 case Intrinsic::aarch64_settag_zero: {
8492 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
8493 bool ZeroMemory =
Intrinsic == Intrinsic::aarch64_settag_zero;
8496 getValue(
I.getArgOperand(1)), MachinePointerInfo(
I.getArgOperand(0)),
8502 case Intrinsic::amdgcn_cs_chain: {
8507 Type *RetTy =
I.getType();
8517 for (
unsigned Idx : {2, 3, 1}) {
8518 TargetLowering::ArgListEntry Arg(
getValue(
I.getOperand(Idx)),
8520 Arg.setAttributes(&
I, Idx);
8521 Args.push_back(Arg);
8524 assert(Args[0].IsInReg &&
"SGPR args should be marked inreg");
8525 assert(!Args[1].IsInReg &&
"VGPR args should not be marked inreg");
8526 Args[2].IsInReg =
true;
8529 for (
unsigned Idx = 4;
Idx <
I.arg_size(); ++
Idx) {
8530 TargetLowering::ArgListEntry Arg(
getValue(
I.getOperand(Idx)),
8532 Arg.setAttributes(&
I, Idx);
8533 Args.push_back(Arg);
8536 TargetLowering::CallLoweringInfo CLI(
DAG);
8539 .setCallee(
CC, RetTy, Callee, std::move(Args))
8542 .setConvergent(
I.isConvergent());
8544 std::pair<SDValue, SDValue>
Result =
8548 "Should've lowered as tail call");
8553 case Intrinsic::amdgcn_call_whole_wave: {
8555 bool isTailCall =
I.isTailCall();
8558 for (
unsigned Idx = 1;
Idx <
I.arg_size(); ++
Idx) {
8559 TargetLowering::ArgListEntry Arg(
getValue(
I.getArgOperand(Idx)),
8560 I.getArgOperand(Idx)->getType());
8561 Arg.setAttributes(&
I, Idx);
8568 Args.push_back(Arg);
8573 auto *Token = Bundle->Inputs[0].get();
8574 ConvControlToken =
getValue(Token);
8577 TargetLowering::CallLoweringInfo CLI(
DAG);
8581 getValue(
I.getArgOperand(0)), std::move(Args))
8585 .setConvergent(
I.isConvergent())
8586 .setConvergenceControlToken(ConvControlToken);
8589 std::pair<SDValue, SDValue>
Result =
8592 if (
Result.first.getNode())
8596 case Intrinsic::ptrmask: {
8612 auto HighOnes =
DAG.getNode(
8613 ISD::SHL, sdl, PtrVT,
DAG.getAllOnesConstant(sdl, PtrVT),
8614 DAG.getShiftAmountConstant(
Mask.getValueType().getFixedSizeInBits(),
8617 DAG.getZExtOrTrunc(Mask, sdl, PtrVT), HighOnes);
8618 }
else if (
Mask.getValueType() != PtrVT)
8619 Mask =
DAG.getPtrExtOrTrunc(Mask, sdl, PtrVT);
8625 case Intrinsic::threadlocal_address: {
8629 case Intrinsic::get_active_lane_mask: {
8633 EVT ElementVT =
Index.getValueType();
8644 SDValue VectorIndex =
DAG.getSplat(VecTy, sdl, Index);
8645 SDValue VectorTripCount =
DAG.getSplat(VecTy, sdl, TripCount);
8646 SDValue VectorStep =
DAG.getStepVector(sdl, VecTy);
8649 SDValue SetCC =
DAG.getSetCC(sdl, CCVT, VectorInduction,
8654 case Intrinsic::mask_beforefirst: {
8660 case Intrinsic::experimental_get_vector_length: {
8662 "Expected positive VF");
8667 EVT CountVT =
Count.getValueType();
8670 visitTargetIntrinsic(
I, Intrinsic);
8679 if (CountVT.
bitsLT(VT)) {
8684 SDValue MaxEVL =
DAG.getElementCount(sdl, CountVT,
8694 case Intrinsic::vector_partial_reduce_add: {
8699 Input,
DAG.getConstant(1, sdl,
Input.getValueType())));
8702 case Intrinsic::vector_partial_reduce_fadd: {
8705 Flags.copyFMF(*FPOp);
8710 DAG.getConstantFP(1.0, sdl,
Input.getValueType()),
8714 case Intrinsic::experimental_cttz_elts: {
8716 EVT OpVT =
Op.getValueType();
8723 SDValue AllZero =
DAG.getConstant(0, sdl, OpVT);
8732 case Intrinsic::vector_insert: {
8740 if (
Index.getValueType() != VectorIdxTy)
8741 Index =
DAG.getVectorIdxConstant(
Index->getAsZExtVal(), sdl);
8748 case Intrinsic::vector_extract: {
8756 if (
Index.getValueType() != VectorIdxTy)
8757 Index =
DAG.getVectorIdxConstant(
Index->getAsZExtVal(), sdl);
8763 case Intrinsic::experimental_vector_match: {
8767 EVT ResVT =
Mask.getValueType();
8771 case Intrinsic::vector_reverse:
8772 visitVectorReverse(
I);
8774 case Intrinsic::vector_splice_left:
8775 case Intrinsic::vector_splice_right:
8776 visitVectorSplice(
I);
8778 case Intrinsic::callbr_landingpad:
8779 visitCallBrLandingPad(
I);
8781 case Intrinsic::vector_interleave2:
8782 visitVectorInterleave(
I, 2);
8784 case Intrinsic::vector_interleave3:
8785 visitVectorInterleave(
I, 3);
8787 case Intrinsic::vector_interleave4:
8788 visitVectorInterleave(
I, 4);
8790 case Intrinsic::vector_interleave5:
8791 visitVectorInterleave(
I, 5);
8793 case Intrinsic::vector_interleave6:
8794 visitVectorInterleave(
I, 6);
8796 case Intrinsic::vector_interleave7:
8797 visitVectorInterleave(
I, 7);
8799 case Intrinsic::vector_interleave8:
8800 visitVectorInterleave(
I, 8);
8802 case Intrinsic::vector_deinterleave2:
8803 visitVectorDeinterleave(
I, 2);
8805 case Intrinsic::vector_deinterleave3:
8806 visitVectorDeinterleave(
I, 3);
8808 case Intrinsic::vector_deinterleave4:
8809 visitVectorDeinterleave(
I, 4);
8811 case Intrinsic::vector_deinterleave5:
8812 visitVectorDeinterleave(
I, 5);
8814 case Intrinsic::vector_deinterleave6:
8815 visitVectorDeinterleave(
I, 6);
8817 case Intrinsic::vector_deinterleave7:
8818 visitVectorDeinterleave(
I, 7);
8820 case Intrinsic::vector_deinterleave8:
8821 visitVectorDeinterleave(
I, 8);
8823 case Intrinsic::vector_repeat: {
8829 case Intrinsic::experimental_vector_compress:
8831 getValue(
I.getArgOperand(0)).getValueType(),
8836 case Intrinsic::vector_shuffle:
8842 case Intrinsic::experimental_convergence_anchor:
8843 case Intrinsic::experimental_convergence_entry:
8844 case Intrinsic::experimental_convergence_loop:
8845 visitConvergenceControl(
I, Intrinsic);
8847 case Intrinsic::experimental_vector_histogram_add: {
8848 visitVectorHistogram(
I, Intrinsic);
8851 case Intrinsic::experimental_vector_extract_last_active: {
8852 visitVectorExtractLastActive(
I, Intrinsic);
8855 case Intrinsic::loop_dependence_war_mask:
8860 DAG.getConstant(0, sdl, MVT::i64)));
8862 case Intrinsic::loop_dependence_raw_mask:
8867 DAG.getConstant(0, sdl, MVT::i64)));
8869 case Intrinsic::masked_udiv:
8875 case Intrinsic::masked_sdiv:
8881 case Intrinsic::masked_urem:
8887 case Intrinsic::masked_srem:
8896void SelectionDAGBuilder::pushFPOpOutChain(
SDValue Result,
8912 PendingConstrainedFP.push_back(OutChain);
8915 PendingConstrainedFPStrict.push_back(OutChain);
8920void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8934 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
8936 SDVTList VTs =
DAG.getVTList(VT, MVT::Other);
8940 Flags.setNoFPExcept(
true);
8943 Flags.copyFMF(*FPOp);
8948#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8949 case Intrinsic::INTRINSIC: \
8950 Opcode = ISD::STRICT_##DAGN; \
8952#include "llvm/IR/ConstrainedOps.def"
8953 case Intrinsic::experimental_constrained_fmuladd: {
8959 pushFPOpOutChain(
Mul, EB);
8982 if (
DAG.isKnownNeverNaN(Opers[1]) &&
DAG.isKnownNeverNaN(Opers[2]))
8990 pushFPOpOutChain(Result, EB);
8997 std::optional<unsigned> ResOPC;
8999 case Intrinsic::vp_cttz_elts: {
9001 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_POISON : ISD::VP_CTTZ_ELTS;
9004#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
9005 case Intrinsic::VPID: \
9006 ResOPC = ISD::VPSD; \
9008#include "llvm/IR/VPIntrinsics.def"
9013 "Inconsistency: no SDNode available for this VPIntrinsic!");
9015 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
9016 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
9018 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
9019 : ISD::VP_REDUCE_FMUL;
9025void SelectionDAGBuilder::visitVPLoad(
9040 SDValue InChain = AddToChain ?
DAG.getRoot() :
DAG.getEntryNode();
9041 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9044 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9045 MachinePointerInfo(PtrOperand), MMOFlags,
9047 MMOMetadata(AAInfo, Ranges));
9048 LD =
DAG.getLoadVP(VT,
DL, InChain, OpValues[0], OpValues[1], OpValues[2],
9055void SelectionDAGBuilder::visitVPLoadFF(
9058 assert(OpValues.
size() == 3 &&
"Unexpected number of operands");
9071 SDValue InChain = AddToChain ?
DAG.getRoot() :
DAG.getEntryNode();
9072 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9075 MMOMetadata(AAInfo, Ranges));
9076 LD =
DAG.getLoadFFVP(VT,
DL, InChain, OpValues[0], OpValues[1], OpValues[2],
9081 setValue(&VPIntrin,
DAG.getMergeValues({LD.getValue(0), Trunc},
DL));
9084void SelectionDAGBuilder::visitVPGather(
9088 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9100 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9102 *Alignment, MMOMetadata(AAInfo, Ranges));
9112 EVT IdxVT =
Index.getValueType();
9118 LD =
DAG.getGatherVP(
9119 DAG.getVTList(VT, MVT::Other), VT,
DL,
9120 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
9126void SelectionDAGBuilder::visitVPStore(
9130 EVT VT = OpValues[0].getValueType();
9138 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9141 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9142 MachinePointerInfo(PtrOperand), MMOFlags,
9151void SelectionDAGBuilder::visitVPScatter(
9154 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9156 EVT VT = OpValues[0].getValueType();
9166 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9168 *Alignment, AAInfo);
9178 EVT IdxVT =
Index.getValueType();
9184 ST =
DAG.getScatterVP(
DAG.getVTList(MVT::Other), VT,
DL,
9185 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
9186 OpValues[2], OpValues[3]},
9192void SelectionDAGBuilder::visitVPStridedLoad(
9204 SDValue InChain = AddToChain ?
DAG.getRoot() :
DAG.getEntryNode();
9206 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9209 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9211 *Alignment, MMOMetadata(AAInfo, Ranges));
9213 SDValue LD =
DAG.getStridedLoadVP(VT,
DL, InChain, OpValues[0], OpValues[1],
9214 OpValues[2], OpValues[3], MMO,
9222void SelectionDAGBuilder::visitVPStridedStore(
9226 EVT VT = OpValues[0].getValueType();
9232 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9235 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9237 *Alignment, AAInfo);
9241 DAG.getPOISON(OpValues[1].getValueType()), OpValues[2], OpValues[3],
9249void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
9257 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9259 SDVTList VTs =
DAG.getVTList(ValueVTs);
9265 "Unexpected target EVL type");
9269 for (
unsigned I = 0;
I < VPIntrin.
arg_size(); ++
I) {
9271 if (
I == EVLParamPos)
9278 SDNodeFlags SDFlags;
9286 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
9288 case ISD::VP_LOAD_FF:
9289 visitVPLoadFF(VPIntrin, ValueVTs[0], ValueVTs[1], OpValues);
9291 case ISD::VP_GATHER:
9292 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
9294 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
9295 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
9298 visitVPStore(VPIntrin, OpValues);
9300 case ISD::VP_SCATTER:
9301 visitVPScatter(VPIntrin, OpValues);
9303 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
9304 visitVPStridedStore(VPIntrin, OpValues);
9306 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
9307 case ISD::VP_CTTZ_ELTS: {
9309 DAG.getNode(Opcode,
DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
9325 BeginLabel =
nullptr;
9335 unsigned CallSiteIndex =
FuncInfo.getCurrentCallSite();
9336 if (CallSiteIndex) {
9366 assert(
II &&
"II should've been set");
9377std::pair<SDValue, SDValue>
9391 std::pair<SDValue, SDValue> Result =
TLI.LowerCallTo(CLI);
9394 "Non-null chain expected with non-tail call!");
9395 assert((Result.second.getNode() || !Result.first.getNode()) &&
9396 "Null value expected with tail call!");
9398 if (!Result.second.getNode()) {
9405 PendingExports.clear();
9407 DAG.setRoot(Result.second);
9425 if (!isMustTailCall &&
9426 Caller->getFnAttribute(
"disable-tail-calls").getValueAsBool())
9432 if (
DAG.hasSwiftErrorArg())
9441 bool isTailCall,
bool isMustTailCall,
9444 auto &
DL =
DAG.getDataLayout();
9451 const Value *SwiftErrorVal =
nullptr;
9458 const Value *V = *
I;
9461 if (V->getType()->isEmptyTy())
9466 Entry.setAttributes(&CB,
I - CB.
arg_begin());
9469 if (Entry.IsSwiftError &&
TLI.supportSwiftError()) {
9478 Args.push_back(Entry);
9489 Value *V = Bundle->Inputs[0];
9491 Entry.IsCFGuardTarget =
true;
9492 Args.push_back(Entry);
9497 if (
TLI.supportSwiftError() && SwiftErrorVal)
9503 if (!
TLI.supportKCFIBundles())
9505 "Target doesn't support calls with kcfi operand bundles.");
9513 auto *Token = Bundle->Inputs[0].get();
9514 ConvControlToken =
getValue(Token);
9525 .
setCallee(RetTy, FTy, Callee, std::move(Args), CB)
9536 if (!
TLI.supportPtrAuthBundles())
9538 "This target doesn't support calls with ptrauth operand bundles.");
9542 std::pair<SDValue, SDValue> Result =
lowerInvokable(CLI, EHPadBB);
9544 if (Result.first.getNode()) {
9553 if (SwiftErrorVal &&
TLI.supportSwiftError()) {
9559 DAG.setRoot(CopyNode);
9575 LoadTy, Builder.DAG.getDataLayout()))
9576 return Builder.getValue(LoadCst);
9582 bool ConstantMemory =
false;
9585 if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(PtrVal)) {
9586 Root = Builder.DAG.getEntryNode();
9587 ConstantMemory =
true;
9590 Root = Builder.DAG.getRoot();
9595 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
9598 if (!ConstantMemory)
9599 Builder.PendingLoads.push_back(LoadVal.
getValue(1));
9605void SelectionDAGBuilder::processIntegerCallValue(
const Instruction &
I,
9608 EVT VT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
9619bool SelectionDAGBuilder::visitMemCmpBCmpCall(
const CallInst &
I) {
9620 const Value *
LHS =
I.getArgOperand(0), *
RHS =
I.getArgOperand(1);
9621 const Value *
Size =
I.getArgOperand(2);
9624 EVT CallVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
9630 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9634 if (Res.first.getNode()) {
9635 processIntegerCallValue(
I, Res.first,
true);
9649 auto hasFastLoadsAndCompare = [&](
unsigned NumBits) {
9650 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9672 switch (NumBitsToCompare) {
9684 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9697 LoadL =
DAG.getBitcast(CmpVT, LoadL);
9698 LoadR =
DAG.getBitcast(CmpVT, LoadR);
9702 processIntegerCallValue(
I, Cmp,
false);
9711bool SelectionDAGBuilder::visitMemChrCall(
const CallInst &
I) {
9712 const Value *Src =
I.getArgOperand(0);
9713 const Value *
Char =
I.getArgOperand(1);
9714 const Value *
Length =
I.getArgOperand(2);
9716 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9717 std::pair<SDValue, SDValue> Res =
9720 MachinePointerInfo(Src));
9721 if (Res.first.getNode()) {
9735bool SelectionDAGBuilder::visitMemCCpyCall(
const CallInst &
I) {
9736 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9743 processIntegerCallValue(
I, Res.first,
true);
9755bool SelectionDAGBuilder::visitMemPCpyCall(
const CallInst &
I) {
9760 Align DstAlign =
DAG.InferPtrAlign(Dst).valueOrOne();
9761 Align SrcAlign =
DAG.InferPtrAlign(Src).valueOrOne();
9770 Root, sdl, Dst, Src,
Size, DstAlign, SrcAlign,
false,
false,
9771 nullptr, std::nullopt, MachinePointerInfo(
I.getArgOperand(0)),
9772 MachinePointerInfo(
I.getArgOperand(1)),
I.getAAMetadata());
9774 "** memcpy should not be lowered as TailCall in mempcpy context **");
9778 Size =
DAG.getSExtOrTrunc(
Size, sdl, Dst.getValueType());
9791bool SelectionDAGBuilder::visitStrCpyCall(
const CallInst &
I,
bool isStpcpy) {
9792 const Value *Arg0 =
I.getArgOperand(0), *Arg1 =
I.getArgOperand(1);
9794 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9797 MachinePointerInfo(Arg0), MachinePointerInfo(Arg1), isStpcpy, &
I);
9798 if (Res.first.getNode()) {
9800 DAG.setRoot(Res.second);
9812bool SelectionDAGBuilder::visitStrCmpCall(
const CallInst &
I) {
9813 const Value *Arg0 =
I.getArgOperand(0), *Arg1 =
I.getArgOperand(1);
9815 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9818 MachinePointerInfo(Arg0), MachinePointerInfo(Arg1), &
I);
9819 if (Res.first.getNode()) {
9820 processIntegerCallValue(
I, Res.first,
true);
9833bool SelectionDAGBuilder::visitStrLenCall(
const CallInst &
I) {
9834 const Value *Arg0 =
I.getArgOperand(0);
9836 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9839 if (Res.first.getNode()) {
9840 processIntegerCallValue(
I, Res.first,
false);
9853bool SelectionDAGBuilder::visitStrNLenCall(
const CallInst &
I) {
9854 const Value *Arg0 =
I.getArgOperand(0), *Arg1 =
I.getArgOperand(1);
9856 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9857 std::pair<SDValue, SDValue> Res =
9860 MachinePointerInfo(Arg0));
9861 if (Res.first.getNode()) {
9862 processIntegerCallValue(
I, Res.first,
false);
9875bool SelectionDAGBuilder::visitStrstrCall(
const CallInst &
I) {
9876 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9877 const Value *Arg0 =
I.getArgOperand(0), *Arg1 =
I.getArgOperand(1);
9881 processIntegerCallValue(
I, Res.first,
false);
9893bool SelectionDAGBuilder::visitUnaryFloatCall(
const CallInst &
I,
9898 if (!
I.onlyReadsMemory() ||
I.isStrictFP())
9915bool SelectionDAGBuilder::visitBinaryFloatCall(
const CallInst &
I,
9920 if (!
I.onlyReadsMemory() ||
I.isStrictFP())
9933void SelectionDAGBuilder::visitCall(
const CallInst &
I) {
9935 if (
I.isInlineAsm()) {
9943 if (
F->isDeclaration()) {
9945 if (
unsigned IID =
F->getIntrinsicID()) {
9946 visitIntrinsicCall(
I, IID);
9956 LibFunc
Func = !
I.isNoBuiltin() && !
F->hasLocalLinkage() &&
F->hasName()
9959 if (
LibInfo->hasOptimizedCodeGen(Func)) {
9963 if (visitMemCmpBCmpCall(
I))
10002 case LibFunc_atan2:
10003 case LibFunc_atan2f:
10004 case LibFunc_atan2l:
10009 case LibFunc_sinhf:
10010 case LibFunc_sinhl:
10015 case LibFunc_coshf:
10016 case LibFunc_coshl:
10021 case LibFunc_tanhf:
10022 case LibFunc_tanhl:
10027 case LibFunc_sqrtf:
10028 case LibFunc_sqrtl:
10029 case LibFunc_sqrt_finite:
10030 case LibFunc_sqrtf_finite:
10031 case LibFunc_sqrtl_finite:
10036 case LibFunc_log2f:
10037 case LibFunc_log2l:
10042 case LibFunc_exp2f:
10043 case LibFunc_exp2l:
10047 case LibFunc_exp10:
10048 case LibFunc_exp10f:
10049 case LibFunc_exp10l:
10053 case LibFunc_strstr:
10054 if (visitStrstrCall(
I))
10057 case LibFunc_memcmp:
10058 if (visitMemCmpBCmpCall(
I))
10061 case LibFunc_memccpy:
10062 if (visitMemCCpyCall(
I))
10065 case LibFunc_mempcpy:
10066 if (visitMemPCpyCall(
I))
10069 case LibFunc_memchr:
10070 if (visitMemChrCall(
I))
10073 case LibFunc_strcpy:
10074 if (visitStrCpyCall(
I,
false))
10077 case LibFunc_stpcpy:
10078 if (visitStrCpyCall(
I,
true))
10081 case LibFunc_strcmp:
10082 if (visitStrCmpCall(
I))
10085 case LibFunc_strlen:
10086 if (visitStrLenCall(
I))
10089 case LibFunc_strnlen:
10090 if (visitStrNLenCall(
I))
10114 if (
I.hasDeoptState())
10131 const Value *Discriminator = PAB->Inputs[1];
10133 assert(
Key->getType()->isIntegerTy(32) &&
"Invalid ptrauth key");
10134 assert(Discriminator->getType()->isIntegerTy(64) &&
10135 "Invalid ptrauth discriminator");
10140 if (CalleeCPA->isKnownCompatibleWith(
Key, Discriminator,
10141 DAG.getDataLayout()))
10181 for (
const auto &Code : Codes)
10196 SDISelAsmOperandInfo &MatchingOpInfo,
10198 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
10204 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
10205 TLI.getRegForInlineAsmConstraint(
TRI, OpInfo.ConstraintCode,
10206 OpInfo.ConstraintVT);
10207 std::pair<unsigned, const TargetRegisterClass *> InputRC =
10208 TLI.getRegForInlineAsmConstraint(
TRI, MatchingOpInfo.ConstraintCode,
10209 MatchingOpInfo.ConstraintVT);
10210 const bool OutOpIsIntOrFP =
10211 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
10212 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
10213 MatchingOpInfo.ConstraintVT.isFloatingPoint();
10214 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
10217 " with a matching output constraint of"
10218 " incompatible type!");
10220 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
10227 SDISelAsmOperandInfo &OpInfo,
10240 const Value *OpVal = OpInfo.CallOperandVal;
10258 DL.getPrefTypeAlign(Ty),
false,
10261 Chain = DAG.
getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
10263 TLI.getMemValueType(
DL, Ty));
10264 OpInfo.CallOperand = StackSlot;
10277static std::optional<unsigned>
10279 SDISelAsmOperandInfo &OpInfo,
10280 SDISelAsmOperandInfo &RefOpInfo) {
10291 return std::nullopt;
10295 unsigned AssignedReg;
10297 std::tie(AssignedReg, RC) =
TLI.getRegForInlineAsmConstraint(
10298 &
TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
10301 return std::nullopt;
10306 const MVT RegVT = *
TRI.legalclasstypes_begin(*RC);
10308 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
10317 !
TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
10322 if (RegVT.
getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
10327 OpInfo.CallOperand =
10329 OpInfo.ConstraintVT = RegVT;
10333 }
else if (RegVT.
isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
10337 return std::nullopt;
10339 OpInfo.CallOperand =
10341 OpInfo.ConstraintVT = VT;
10348 if (OpInfo.isMatchingInputConstraint())
10349 return std::nullopt;
10351 EVT ValueVT = OpInfo.ConstraintVT;
10352 if (OpInfo.ConstraintVT == MVT::Other)
10356 unsigned NumRegs = 1;
10357 if (OpInfo.ConstraintVT != MVT::Other)
10358 NumRegs =
TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
10372 I = std::find(
I, RC->
end(), AssignedReg);
10373 if (
I == RC->
end()) {
10376 return {AssignedReg};
10380 for (; NumRegs; --NumRegs, ++
I) {
10381 assert(
I != RC->
end() &&
"Ran out of registers to allocate!");
10386 OpInfo.AssignedRegs =
RegsForValue(Regs, RegVT, ValueVT);
10387 return std::nullopt;
10392 const std::vector<SDValue> &AsmNodeOperands) {
10395 for (; OperandNo; --OperandNo) {
10397 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
10400 (
F.isRegDefKind() ||
F.isRegDefEarlyClobberKind() ||
F.isMemKind()) &&
10401 "Skipped past definitions?");
10402 CurOp +=
F.getNumOperandRegisters() + 1;
10410 unsigned Flags = 0;
10413 explicit ExtraFlags(
const CallBase &
Call) {
10415 if (
IA->hasSideEffects())
10417 if (
IA->isAlignStack())
10419 if (
IA->canThrow())
10426 void update(
const TargetLowering::AsmOperandInfo &OpInfo) {
10442 unsigned get()
const {
return Flags; }
10466struct ConstraintDecisionInfo {
10468 std::vector<SDValue> AsmNodeOperands;
10470 bool HasSideEffect =
false;
10473 SmallVector<char> Buffer;
10474 raw_svector_ostream ErrorMsg;
10476 ConstraintDecisionInfo() : ErrorMsg(Buffer) {}
10486 ExtraFlags &ExtraInfo) {
10487 for (
auto &
T : TargetConstraints) {
10488 Info.ConstraintOperands.push_back(SDISelAsmOperandInfo(
T));
10489 SDISelAsmOperandInfo &OpInfo = Info.ConstraintOperands.back();
10491 if (OpInfo.CallOperandVal)
10492 OpInfo.CallOperand = Builder.getValue(OpInfo.CallOperandVal);
10494 if (!Info.HasSideEffect)
10495 Info.HasSideEffect = OpInfo.hasMemory(
TLI);
10507 Info.ErrorMsg <<
"constraint '" <<
T.ConstraintCode
10508 <<
"' expects an integer constant expression";
10512 ExtraInfo.update(
T);
10526 IA->collectAsmStrs(AsmStrs);
10529 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10537 if (OpInfo.hasMatchingInput()) {
10538 SDISelAsmOperandInfo &
Input =
10539 Info.ConstraintOperands[OpInfo.MatchingInput];
10544 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
10570 if (OpInfo.isIndirect &&
isFunction(OpInfo.CallOperand) &&
10571 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
10573 OpInfo.isIndirect =
false;
10580 !OpInfo.isIndirect) {
10581 assert((OpInfo.isMultipleAlternative ||
10583 "Can only indirectify direct input operands!");
10590 OpInfo.CallOperandVal =
nullptr;
10593 OpInfo.isIndirect =
true;
10605 SDLoc DL = Builder.getCurSDLoc();
10606 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10608 SDISelAsmOperandInfo &RefOpInfo =
10609 OpInfo.isMatchingInputConstraint()
10610 ? Info.ConstraintOperands[OpInfo.getMatchedOperand()]
10616 const char *
RegName =
TRI.getName(*RegError);
10617 Info.ErrorMsg <<
"register '" <<
RegName <<
"' allocated for constraint '"
10618 << OpInfo.ConstraintCode
10619 <<
"' does not match required type";
10623 auto DetectWriteToReservedRegister = [&]() {
10628 if (
Reg.isPhysical() &&
TRI.isInlineAsmReadOnlyReg(MF,
Reg)) {
10629 Info.ErrorMsg <<
"write to reserved register '"
10630 <<
TRI.getRegAsmName(
Reg) <<
"'";
10639 !OpInfo.isMatchingInputConstraint())) &&
10640 "Only address as input operand is allowed.");
10642 switch (OpInfo.Type) {
10646 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10648 "Failed to convert memory constraint code to constraint id.");
10653 Info.AsmNodeOperands.push_back(
10655 Info.AsmNodeOperands.push_back(OpInfo.CallOperand);
10660 if (OpInfo.AssignedRegs.Regs.empty()) {
10661 Info.ErrorMsg <<
"could not allocate output register for "
10662 <<
"constraint '" << OpInfo.ConstraintCode <<
"'";
10666 if (DetectWriteToReservedRegister())
10671 OpInfo.AssignedRegs.AddInlineAsmOperands(
10674 false, 0,
DL, DAG, Info.AsmNodeOperands);
10680 SDValue InOperandVal = OpInfo.CallOperand;
10682 if (OpInfo.isMatchingInputConstraint()) {
10686 Info.AsmNodeOperands);
10688 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10689 if (OpInfo.isIndirect) {
10691 Info.ErrorMsg <<
"inline asm not supported yet: cannot handle "
10692 <<
"tied indirect register inputs";
10702 MVT RegVT = R->getSimpleValueType(0);
10705 RC =
MRI.getRegClass(TiedReg);
10706 }
else if (RegVT == MVT::Untyped) {
10707 RC =
TRI.getMinimalPhysRegClass(TiedReg);
10709 RC =
TLI.getRegClassFor(RegVT);
10712 TRI.getMinimalPhysRegClass(TiedReg);
10718 for (
unsigned I = 0,
E = Flag.getNumOperandRegisters();
I !=
E; ++
I)
10725 &Info.Glue, &
Call);
10727 OpInfo.getMatchedOperand(),
DL, DAG,
10728 Info.AsmNodeOperands);
10732 assert(Flag.isMemKind() &&
"Unknown matching constraint!");
10733 assert(Flag.getNumOperandRegisters() == 1 &&
10734 "Unexpected number of operands");
10738 Flag.clearMemConstraint();
10739 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10742 Info.AsmNodeOperands.push_back(Info.AsmNodeOperands[CurOp + 1]);
10753 std::vector<SDValue>
Ops;
10754 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
10759 Info.ErrorMsg <<
"value out of range for constraint '"
10760 << OpInfo.ConstraintCode <<
"'";
10764 Info.ErrorMsg <<
"invalid operand for inline asm constraint '"
10765 << OpInfo.ConstraintCode <<
"'";
10778 assert((OpInfo.isIndirect ||
10780 "Operand must be indirect to be a mem!");
10785 OpInfo.CallOperandVal
10787 ->getAddressSpace()
10789 "Memory operands expect pointer values");
10792 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10794 "Failed to convert memory constraint code to constraint id.");
10799 Info.AsmNodeOperands.push_back(
10801 Info.AsmNodeOperands.push_back(InOperandVal);
10807 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10809 "Failed to convert memory constraint code to constraint id.");
10813 SDValue AsmOp = InOperandVal;
10814 if (
TLI.supportsInlineAsmFuncOperands() &&
isFunction(InOperandVal)) {
10825 Info.AsmNodeOperands.push_back(
10827 Info.AsmNodeOperands.push_back(AsmOp);
10833 Info.ErrorMsg <<
"unknown asm constraint '" << OpInfo.ConstraintCode
10839 if (OpInfo.isIndirect) {
10840 Info.ErrorMsg <<
"cannot handle indirect register inputs yet for "
10841 <<
"constraint '" << OpInfo.ConstraintCode <<
"'";
10846 if (OpInfo.AssignedRegs.Regs.empty()) {
10847 Info.ErrorMsg <<
"could not allocate input reg for constraint '"
10848 << OpInfo.ConstraintCode <<
"'";
10852 if (DetectWriteToReservedRegister())
10855 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG,
DL, Info.Chain,
10856 &Info.Glue, &
Call);
10857 OpInfo.AssignedRegs.AddInlineAsmOperands(
10865 if (!OpInfo.AssignedRegs.Regs.empty())
10866 OpInfo.AssignedRegs.AddInlineAsmOperands(
10883 ExtraFlags ExtraInfo(
Call);
10886 Info.HasSideEffect = IA->hasSideEffects();
10892 Info.Chain = Info.HasSideEffect ? Builder.getRoot() : DAG.
getRoot();
10896 if (IsCallBr || EmitEHLabels)
10900 Info.Chain = Builder.getControlRoot();
10903 Info.Chain = Builder.lowerStartEH(Info.Chain, EHPadBB, Info.BeginLabel);
10909 Info.AsmNodeOperands.push_back(
SDValue());
10911 IA->getAsmString().data(),
TLI.getProgramPointerTy(DAG.
getDataLayout())));
10916 const MDNode *SrcLoc =
Call.getMetadata(
"srcloc");
10917 Info.AsmNodeOperands.push_back(DAG.
getMDNode(SrcLoc));
10921 Info.AsmNodeOperands.push_back(
10930void SelectionDAGBuilder::visitInlineAsm(
const CallBase &
Call,
10932 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
10934 DAG.getDataLayout(),
DAG.getSubtarget().getRegisterInfo(),
Call);
10937 "InvokeInst must have an EHPadBB");
10939 ConstraintDecisionInfo
Info;
10942 return emitInlineAsmError(
Call,
Info.ErrorMsg.str());
10950 Info.AsmNodeOperands.push_back(Glue);
10956 Info.AsmNodeOperands);
10968 ResultTypes = StructResult->elements();
10969 else if (!CallResultType->
isVoidTy())
10970 ResultTypes =
ArrayRef(CallResultType);
10972 auto CurResultType = ResultTypes.
begin();
10973 auto handleRegAssign = [&](
SDValue V) {
10974 assert(CurResultType != ResultTypes.
end() &&
"Unexpected value");
10975 assert((*CurResultType)->isSized() &&
"Unexpected unsized type");
10988 if (ResultVT !=
V.getValueType() &&
10991 else if (ResultVT !=
V.getValueType() && ResultVT.
isInteger() &&
10992 V.getValueType().isInteger()) {
10998 assert(ResultVT ==
V.getValueType() &&
"Asm result value mismatch!");
11004 for (SDISelAsmOperandInfo &OpInfo :
Info.ConstraintOperands) {
11008 if (OpInfo.AssignedRegs.
Regs.empty())
11011 switch (OpInfo.ConstraintType) {
11015 Chain, &Glue, &
Call);
11027 assert(
false &&
"Unexpected unknown constraint");
11031 if (OpInfo.isIndirect) {
11032 const Value *Ptr = OpInfo.CallOperandVal;
11033 assert(Ptr &&
"Expected value CallOperandVal for indirect asm operand");
11035 MachinePointerInfo(Ptr));
11042 handleRegAssign(
V);
11044 handleRegAssign(Val);
11050 if (!ResultValues.
empty()) {
11051 assert(CurResultType == ResultTypes.
end() &&
11052 "Mismatch in number of ResultTypes");
11054 "Mismatch in number of output operands in asm result");
11057 DAG.getVTList(ResultVTs), ResultValues);
11062 if (!OutChains.
empty())
11066 Chain = lowerEndEH(Chain,
II, EHPadBB,
Info.BeginLabel);
11069 if (ResultValues.
empty() ||
Info.HasSideEffect || !OutChains.
empty() ||
11071 DAG.setRoot(Chain);
11074void SelectionDAGBuilder::emitInlineAsmError(
const CallBase &
Call,
11075 const Twine &Message) {
11076 LLVMContext &Ctx = *
DAG.getContext();
11080 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
11084 if (ValueVTs.
empty())
11088 for (
const EVT &VT : ValueVTs)
11089 Ops.push_back(
DAG.getUNDEF(VT));
11094void SelectionDAGBuilder::visitVAStart(
const CallInst &
I) {
11098 DAG.getSrcValue(
I.getArgOperand(0))));
11101void SelectionDAGBuilder::visitVAArg(
const VAArgInst &
I) {
11102 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
11103 const DataLayout &
DL =
DAG.getDataLayout();
11107 DL.getABITypeAlign(
I.getType()).value());
11108 DAG.setRoot(
V.getValue(1));
11110 if (
I.getType()->isPointerTy())
11111 V =
DAG.getPtrExtOrTrunc(
11116void SelectionDAGBuilder::visitVAEnd(
const CallInst &
I) {
11120 DAG.getSrcValue(
I.getArgOperand(0))));
11123void SelectionDAGBuilder::visitVACopy(
const CallInst &
I) {
11128 DAG.getSrcValue(
I.getArgOperand(0)),
11129 DAG.getSrcValue(
I.getArgOperand(1))));
11135 std::optional<ConstantRange> CR =
getRange(
I);
11137 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
11140 APInt Hi = CR->getUnsignedMax();
11141 unsigned Bits = std::max(
Hi.getActiveBits(),
11149 DAG.getValueType(SmallVT));
11150 unsigned NumVals =
Op.getNode()->getNumValues();
11156 Ops.push_back(ZExt);
11157 for (
unsigned I = 1;
I != NumVals; ++
I)
11158 Ops.push_back(
Op.getValue(
I));
11160 return DAG.getMergeValues(
Ops,
SL);
11170 SDValue TestConst =
DAG.getTargetConstant(Classes,
SDLoc(), MVT::i32);
11178 for (
unsigned I = 0, E =
Ops.size();
I != E; ++
I) {
11181 MergeOp, TestConst);
11184 return DAG.getMergeValues(
Ops,
SL);
11195 unsigned ArgIdx,
unsigned NumArgs,
SDValue Callee,
Type *ReturnTy,
11198 Args.reserve(NumArgs);
11202 for (
unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
11203 ArgI != ArgE; ++ArgI) {
11204 const Value *V =
Call->getOperand(ArgI);
11206 assert(!V->getType()->isEmptyTy() &&
"Empty type passed to intrinsic.");
11209 Entry.setAttributes(
Call, ArgI);
11210 Args.push_back(Entry);
11215 .
setCallee(
Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
11244 for (
unsigned I = StartIdx;
I <
Call.arg_size();
I++) {
11253 Ops.push_back(Builder.getValue(
Call.getArgOperand(
I)));
11259void SelectionDAGBuilder::visitStackmap(
const CallInst &CI) {
11285 Ops.push_back(Chain);
11286 Ops.push_back(InGlue);
11293 assert(
ID.getValueType() == MVT::i64);
11295 DAG.getTargetConstant(
ID->getAsZExtVal(),
DL,
ID.getValueType());
11296 Ops.push_back(IDConst);
11302 Ops.push_back(ShadConst);
11308 SDVTList NodeTys =
DAG.getVTList(MVT::Other, MVT::Glue);
11312 Chain =
DAG.getCALLSEQ_END(Chain, 0, 0, InGlue,
DL);
11317 DAG.setRoot(Chain);
11320 FuncInfo.MF->getFrameInfo().setHasStackMap();
11324void SelectionDAGBuilder::visitPatchpoint(
const CallBase &CB,
11341 Callee =
DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
11344 Callee =
DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
11345 SDLoc(SymbolicCallee),
11346 SymbolicCallee->getValueType(0));
11356 "Not enough arguments provided to the patchpoint intrinsic");
11359 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
11363 TargetLowering::CallLoweringInfo CLI(
DAG);
11368 SDNode *CallEnd =
Result.second.getNode();
11377 "Expected a callseq node.");
11379 bool HasGlue =
Call->getGluedNode();
11404 Ops.push_back(Callee);
11410 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
11411 Ops.push_back(
DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
11414 Ops.push_back(
DAG.getTargetConstant((
unsigned)
CC, dl, MVT::i32));
11419 for (
unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i !=
e; ++i)
11430 if (IsAnyRegCC && HasDef) {
11432 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
11435 assert(ValueVTs.
size() == 1 &&
"Expected only one return value type.");
11440 NodeTys =
DAG.getVTList(ValueVTs);
11442 NodeTys =
DAG.getVTList(MVT::Other, MVT::Glue);
11459 if (IsAnyRegCC && HasDef) {
11462 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
11468 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
11471void SelectionDAGBuilder::visitVectorReduce(
const CallInst &
I,
11473 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
11476 if (
I.arg_size() > 1)
11481 SDNodeFlags SDFlags;
11485 switch (Intrinsic) {
11486 case Intrinsic::vector_reduce_fadd:
11494 case Intrinsic::vector_reduce_fmul:
11502 case Intrinsic::vector_reduce_add:
11505 case Intrinsic::vector_reduce_mul:
11508 case Intrinsic::vector_reduce_and:
11511 case Intrinsic::vector_reduce_or:
11514 case Intrinsic::vector_reduce_xor:
11517 case Intrinsic::vector_reduce_smax:
11520 case Intrinsic::vector_reduce_smin:
11523 case Intrinsic::vector_reduce_umax:
11526 case Intrinsic::vector_reduce_umin:
11529 case Intrinsic::vector_reduce_fmax:
11532 case Intrinsic::vector_reduce_fmin:
11535 case Intrinsic::vector_reduce_fmaximum:
11538 case Intrinsic::vector_reduce_fminimum:
11541 case Intrinsic::vector_reduce_fmaximumnum:
11544 case Intrinsic::vector_reduce_fminimumnum:
11558 Attrs.push_back(Attribute::SExt);
11560 Attrs.push_back(Attribute::ZExt);
11562 Attrs.push_back(Attribute::InReg);
11564 return AttributeList::get(CLI.
RetTy->
getContext(), AttributeList::ReturnIndex,
11572std::pair<SDValue, SDValue>
11586 "Only supported for non-aggregate returns");
11589 for (
Type *Ty : RetOrigTys)
11598 RetOrigTys.
swap(OldRetOrigTys);
11599 RetVTs.
swap(OldRetVTs);
11600 Offsets.swap(OldOffsets);
11602 for (
size_t i = 0, e = OldRetVTs.
size(); i != e; ++i) {
11603 EVT RetVT = OldRetVTs[i];
11604 uint64_t
Offset = OldOffsets[i];
11607 unsigned RegisterVTByteSZ = RegisterVT.
getSizeInBits() / 8;
11608 RetOrigTys.
append(NumRegs, OldRetOrigTys[i]);
11609 RetVTs.
append(NumRegs, RegisterVT);
11610 for (
unsigned j = 0; j != NumRegs; ++j)
11623 int DemoteStackIdx = -100;
11628 uint64_t TySize =
DL.getTypeAllocSize(CLI.
RetTy);
11636 ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);
11637 Entry.IsSRet =
true;
11638 Entry.Alignment = Alignment;
11650 for (
unsigned I = 0, E = RetVTs.
size();
I != E; ++
I) {
11652 if (NeedsRegBlock) {
11653 Flags.setInConsecutiveRegs();
11654 if (
I == RetVTs.
size() - 1)
11655 Flags.setInConsecutiveRegsLast();
11657 EVT VT = RetVTs[
I];
11661 for (
unsigned i = 0; i != NumRegs; ++i) {
11675 CLI.
Ins.push_back(Ret);
11684 if (Arg.IsSwiftError) {
11690 CLI.
Ins.push_back(Ret);
11698 for (
unsigned i = 0, e = Args.size(); i != e; ++i) {
11702 Type *FinalType = Args[i].Ty;
11703 if (Args[i].IsByVal)
11704 FinalType = Args[i].IndirectType;
11707 for (
unsigned Value = 0, NumValues = OrigArgTys.
size();
Value != NumValues;
11710 Type *ArgTy = OrigArgTy;
11711 if (Args[i].Ty != Args[i].OrigTy) {
11712 assert(
Value == 0 &&
"Only supported for non-aggregate arguments");
11713 ArgTy = Args[i].Ty;
11718 Args[i].Node.getResNo() +
Value);
11725 Flags.setOrigAlign(OriginalAlignment);
11730 Flags.setPointer();
11733 if (Args[i].IsZExt)
11735 if (Args[i].IsSExt)
11737 if (Args[i].IsNoExt)
11739 if (Args[i].IsInReg) {
11746 Flags.setHvaStart();
11752 if (Args[i].IsSRet)
11754 if (Args[i].IsSwiftSelf)
11755 Flags.setSwiftSelf();
11756 if (Args[i].IsSwiftAsync)
11757 Flags.setSwiftAsync();
11758 if (Args[i].IsSwiftError)
11759 Flags.setSwiftError();
11760 if (Args[i].IsCFGuardTarget)
11761 Flags.setCFGuardTarget();
11762 if (Args[i].IsByVal)
11764 if (Args[i].IsByRef)
11766 if (Args[i].IsPreallocated) {
11767 Flags.setPreallocated();
11775 if (Args[i].IsInAlloca) {
11776 Flags.setInAlloca();
11785 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11786 unsigned FrameSize =
DL.getTypeAllocSize(Args[i].IndirectType);
11787 Flags.setByValSize(FrameSize);
11790 if (
auto MA = Args[i].Alignment)
11794 }
else if (
auto MA = Args[i].Alignment) {
11797 MemAlign = OriginalAlignment;
11799 Flags.setMemAlign(MemAlign);
11800 if (Args[i].IsNest)
11803 Flags.setInConsecutiveRegs();
11811 if (Args[i].IsSExt)
11813 else if (Args[i].IsZExt)
11818 if (Args[i].IsReturned && !
Op.getValueType().isVector() &&
11823 Args[i].Ty->getPointerAddressSpace())) &&
11824 RetVTs.
size() == NumValues &&
"unexpected use of 'returned'");
11837 CLI.
RetZExt == Args[i].IsZExt))
11838 Flags.setReturned();
11844 for (
unsigned j = 0; j !=
NumParts; ++j) {
11850 j * Parts[j].
getValueType().getStoreSize().getKnownMinValue());
11859 CLI.
Outs.push_back(MyFlags);
11860 CLI.
OutVals.push_back(Parts[j]);
11863 if (NeedsRegBlock &&
Value == NumValues - 1)
11864 CLI.
Outs[CLI.
Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11876 "LowerCall didn't return a valid chain!");
11878 "LowerCall emitted a return value for a tail call!");
11880 "LowerCall didn't emit the correct number of values!");
11892 for (
unsigned i = 0, e = CLI.
Ins.size(); i != e; ++i) {
11893 assert(InVals[i].
getNode() &&
"LowerCall emitted a null value!");
11894 assert(
EVT(CLI.
Ins[i].VT) == InVals[i].getValueType() &&
11895 "LowerCall emitted a value with the wrong type!");
11905 unsigned NumValues = RetVTs.
size();
11906 ReturnValues.
resize(NumValues);
11913 for (
unsigned i = 0; i < NumValues; ++i) {
11920 DemoteStackIdx, Offsets[i]),
11922 ReturnValues[i] = L;
11923 Chains[i] = L.getValue(1);
11930 std::optional<ISD::NodeType> AssertOp;
11935 unsigned CurReg = 0;
11936 for (
EVT VT : RetVTs) {
11942 CLI.
DAG, CLI.
DL, &InVals[CurReg], NumRegs, RegisterVT, VT,
nullptr,
11950 if (ReturnValues.
empty())
11956 return std::make_pair(Res, CLI.
Chain);
11973 if (
N->getNumValues() == 1) {
11981 "Lowering returned the wrong number of results!");
11984 for (
unsigned I = 0, E =
N->getNumValues();
I != E; ++
I)
11998 "Copy from a reg to the same reg!");
11999 assert(!Reg.isPhysical() &&
"Is a physreg");
12010 auto PreferredExtendIt =
FuncInfo.PreferredExtendType.find(V);
12011 if (PreferredExtendIt !=
FuncInfo.PreferredExtendType.end())
12012 ExtendType = PreferredExtendIt->second;
12015 PendingExports.push_back(Chain);
12027 return A->use_empty();
12029 const BasicBlock &Entry =
A->getParent()->front();
12030 for (
const User *U :
A->users())
12039 std::pair<const AllocaInst *, const StoreInst *>>;
12051 enum StaticAllocaInfo {
Unknown, Clobbered, Elidable };
12053 unsigned NumArgs = FuncInfo->
Fn->
arg_size();
12054 StaticAllocas.
reserve(NumArgs * 2);
12056 auto GetInfoIfStaticAlloca = [&](
const Value *
V) -> StaticAllocaInfo * {
12059 V =
V->stripPointerCasts();
12061 if (!AI || !AI->isStaticAlloca() || !FuncInfo->
StaticAllocaMap.count(AI))
12064 return &Iter.first->second;
12084 for (
const Use &U :
I.operands()) {
12085 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
12086 *Info = StaticAllocaInfo::Clobbered;
12092 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(
SI->getValueOperand()))
12093 *Info = StaticAllocaInfo::Clobbered;
12096 const Value *Dst =
SI->getPointerOperand()->stripPointerCasts();
12097 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
12103 if (*Info != StaticAllocaInfo::Unknown)
12111 const Value *Val =
SI->getValueOperand()->stripPointerCasts();
12114 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
12116 DL.getTypeStoreSize(Arg->
getType()) != *AllocaSize ||
12117 !
DL.typeSizeEqualsStoreSize(Arg->
getType()) ||
12118 ArgCopyElisionCandidates.count(Arg)) {
12119 *Info = StaticAllocaInfo::Clobbered;
12123 LLVM_DEBUG(
dbgs() <<
"Found argument copy elision candidate: " << *AI
12127 *Info = StaticAllocaInfo::Elidable;
12128 ArgCopyElisionCandidates.insert({Arg, {AI,
SI}});
12133 if (ArgCopyElisionCandidates.size() == NumArgs)
12157 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
12158 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
12159 const AllocaInst *AI = ArgCopyIter->second.first;
12160 int FixedIndex = FINode->getIndex();
12162 int OldIndex = AllocaIndex;
12166 dbgs() <<
" argument copy elision failed due to bad fixed stack "
12172 LLVM_DEBUG(
dbgs() <<
" argument copy elision failed: alignment of alloca "
12173 "greater than stack argument alignment ("
12174 <<
DebugStr(RequiredAlignment) <<
" vs "
12182 dbgs() <<
"Eliding argument copy from " << Arg <<
" to " << *AI <<
'\n'
12183 <<
" Replacing frame index " << OldIndex <<
" with " << FixedIndex
12189 AllocaIndex = FixedIndex;
12190 ArgCopyElisionFrameIndexMap.
insert({OldIndex, FixedIndex});
12191 for (
SDValue ArgVal : ArgVals)
12195 const StoreInst *
SI = ArgCopyIter->second.second;
12209 SelectionDAG &DAG =
SDB->DAG;
12210 SDLoc dl =
SDB->getCurSDLoc();
12215 if (
F.hasFnAttribute(Attribute::Naked))
12220 MVT ValueVT =
TLI->getPointerTy(
DL,
DL.getAllocaAddrSpace());
12222 ISD::ArgFlagsTy
Flags;
12224 MVT RegisterVT =
TLI->getRegisterType(*DAG.
getContext(), ValueVT);
12225 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT,
F.getReturnType(),
true,
12235 ArgCopyElisionCandidates);
12238 for (
const Argument &Arg :
F.args()) {
12239 unsigned ArgNo = Arg.getArgNo();
12242 bool isArgValueUsed = !Arg.
use_empty();
12244 if (Arg.hasAttribute(Attribute::ByVal))
12245 FinalType = Arg.getParamByValType();
12246 bool NeedsRegBlock =
TLI->functionArgumentNeedsConsecutiveRegisters(
12247 FinalType,
F.getCallingConv(),
F.isVarArg(),
DL);
12248 for (
unsigned Value = 0, NumValues =
Types.size();
Value != NumValues;
12251 EVT VT =
TLI->getValueType(
DL, ArgTy);
12252 ISD::ArgFlagsTy
Flags;
12255 Flags.setPointer();
12258 if (Arg.hasAttribute(Attribute::ZExt))
12260 if (Arg.hasAttribute(Attribute::SExt))
12262 if (Arg.hasAttribute(Attribute::InReg)) {
12269 Flags.setHvaStart();
12275 if (Arg.hasAttribute(Attribute::StructRet))
12277 if (Arg.hasAttribute(Attribute::SwiftSelf))
12278 Flags.setSwiftSelf();
12279 if (Arg.hasAttribute(Attribute::SwiftAsync))
12280 Flags.setSwiftAsync();
12281 if (Arg.hasAttribute(Attribute::SwiftError))
12282 Flags.setSwiftError();
12283 if (Arg.hasAttribute(Attribute::ByVal))
12285 if (Arg.hasAttribute(Attribute::ByRef))
12287 if (Arg.hasAttribute(Attribute::InAlloca)) {
12288 Flags.setInAlloca();
12296 if (Arg.hasAttribute(Attribute::Preallocated)) {
12297 Flags.setPreallocated();
12309 const Align OriginalAlignment(
12310 TLI->getABIAlignmentForCallingConv(ArgTy,
DL));
12311 Flags.setOrigAlign(OriginalAlignment);
12314 Type *ArgMemTy =
nullptr;
12315 if (
Flags.isByVal() ||
Flags.isInAlloca() ||
Flags.isPreallocated() ||
12318 ArgMemTy = Arg.getPointeeInMemoryValueType();
12320 uint64_t MemSize =
DL.getTypeAllocSize(ArgMemTy);
12325 if (
auto ParamAlign = Arg.getParamStackAlign())
12326 MemAlign = *ParamAlign;
12327 else if ((ParamAlign = Arg.getParamAlign()))
12328 MemAlign = *ParamAlign;
12330 MemAlign =
TLI->getByValTypeAlignment(ArgMemTy,
DL);
12331 if (
Flags.isByRef())
12332 Flags.setByRefSize(MemSize);
12334 Flags.setByValSize(MemSize);
12335 }
else if (
auto ParamAlign = Arg.getParamStackAlign()) {
12336 MemAlign = *ParamAlign;
12338 MemAlign = OriginalAlignment;
12340 Flags.setMemAlign(MemAlign);
12342 if (Arg.hasAttribute(Attribute::Nest))
12345 Flags.setInConsecutiveRegs();
12346 if (ArgCopyElisionCandidates.count(&Arg))
12347 Flags.setCopyElisionCandidate();
12348 if (Arg.hasAttribute(Attribute::Returned))
12349 Flags.setReturned();
12351 MVT RegisterVT =
TLI->getRegisterTypeForCallingConv(
12352 *
CurDAG->getContext(),
F.getCallingConv(), VT);
12353 unsigned NumRegs =
TLI->getNumRegistersForCallingConv(
12354 *
CurDAG->getContext(),
F.getCallingConv(), VT);
12355 for (
unsigned i = 0; i != NumRegs; ++i) {
12359 ISD::InputArg MyFlags(
12360 Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,
12362 if (NumRegs > 1 && i == 0)
12363 MyFlags.Flags.setSplit();
12366 MyFlags.Flags.setOrigAlign(
Align(1));
12367 if (i == NumRegs - 1)
12368 MyFlags.Flags.setSplitEnd();
12372 if (NeedsRegBlock &&
Value == NumValues - 1)
12373 Ins[Ins.
size() - 1].Flags.setInConsecutiveRegsLast();
12379 SDValue NewRoot =
TLI->LowerFormalArguments(
12380 DAG.
getRoot(),
F.getCallingConv(),
F.isVarArg(), Ins, dl, DAG, InVals);
12384 "LowerFormalArguments didn't return a valid chain!");
12386 "LowerFormalArguments didn't emit the correct number of values!");
12388 "LowerFormalArguments emitted a null value!");
12398 MVT VT =
TLI->getPointerTy(
DL,
DL.getAllocaAddrSpace());
12399 MVT RegVT =
TLI->getRegisterType(*
CurDAG->getContext(), VT);
12400 std::optional<ISD::NodeType> AssertOp;
12403 F.getCallingConv(), AssertOp);
12406 MachineRegisterInfo&
RegInfo =
MF.getRegInfo();
12408 RegInfo.createVirtualRegister(
TLI->getRegClassFor(RegVT));
12409 FuncInfo->DemoteRegister = SRetReg;
12411 SDB->DAG.getCopyToReg(NewRoot,
SDB->getCurSDLoc(), SRetReg, ArgValue);
12419 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
12420 for (
const Argument &Arg :
F.args()) {
12424 unsigned NumValues = ValueVTs.
size();
12425 if (NumValues == 0)
12432 if (Ins[i].
Flags.isCopyElisionCandidate()) {
12434 for (EVT VT : ValueVTs)
12436 F.getCallingConv(), VT);
12445 bool isSwiftErrorArg =
12446 TLI->supportSwiftError() &&
12447 Arg.hasAttribute(Attribute::SwiftError);
12448 if (!ArgHasUses && !isSwiftErrorArg) {
12449 SDB->setUnusedArgValue(&Arg, InVals[i]);
12452 if (FrameIndexSDNode *FI =
12454 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12457 for (
unsigned Val = 0; Val != NumValues; ++Val) {
12458 EVT VT = ValueVTs[Val];
12459 MVT PartVT =
TLI->getRegisterTypeForCallingConv(*
CurDAG->getContext(),
12460 F.getCallingConv(), VT);
12461 unsigned NumParts =
TLI->getNumRegistersForCallingConv(
12462 *
CurDAG->getContext(),
F.getCallingConv(), VT);
12467 if (ArgHasUses || isSwiftErrorArg) {
12468 std::optional<ISD::NodeType> AssertOp;
12469 if (Arg.hasAttribute(Attribute::SExt))
12471 else if (Arg.hasAttribute(Attribute::ZExt))
12476 NewRoot,
F.getCallingConv(), AssertOp);
12479 if (NoFPClass !=
fcNone) {
12481 static_cast<uint64_t>(NoFPClass), dl, MVT::i32);
12483 OutVal, SDNoFPClass);
12492 if (ArgValues.
empty())
12496 if (FrameIndexSDNode *FI =
12498 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12501 SDB->getCurSDLoc());
12503 SDB->setValue(&Arg, Res);
12513 if (LoadSDNode *LNode =
12515 if (FrameIndexSDNode *FI =
12517 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12545 FuncInfo->InitializeRegForValue(&Arg);
12546 SDB->CopyToExportRegsIfNeeded(&Arg);
12550 if (!Chains.
empty()) {
12557 assert(i == InVals.
size() &&
"Argument register count mismatch!");
12561 if (!ArgCopyElisionFrameIndexMap.
empty()) {
12562 for (MachineFunction::VariableDbgInfo &VI :
12563 MF->getInStackSlotVariableDbgInfo()) {
12564 auto I = ArgCopyElisionFrameIndexMap.
find(
VI.getStackSlot());
12565 if (
I != ArgCopyElisionFrameIndexMap.
end())
12566 VI.updateStackSlot(
I->second);
12581SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(
const BasicBlock *LLVMBB) {
12582 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
12584 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
12590 MachineBasicBlock *SuccMBB =
FuncInfo.getMBB(SuccBB);
12594 if (!SuccsHandled.
insert(SuccMBB).second)
12602 for (
const PHINode &PN : SuccBB->phis()) {
12604 if (PN.use_empty())
12608 if (PN.getType()->isEmptyTy())
12612 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
12617 RegOut =
FuncInfo.CreateRegs(PHIOp);
12628 auto I =
FuncInfo.ValueMap.find(PHIOp);
12634 "Didn't codegen value into a register!??");
12644 for (EVT VT : ValueVTs) {
12646 for (
unsigned i = 0; i != NumRegisters; ++i)
12648 Reg += NumRegisters;
12668void SelectionDAGBuilder::updateDAGForMaybeTailCall(
SDValue MaybeTC) {
12670 if (MaybeTC.
getNode() !=
nullptr)
12671 DAG.setRoot(MaybeTC);
12676void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W,
Value *
Cond,
12680 MachineBasicBlock *NextMBB =
nullptr;
12685 unsigned Size =
W.LastCluster -
W.FirstCluster + 1;
12687 BranchProbabilityInfo *BPI =
FuncInfo.BPI;
12689 if (
Size == 2 &&
W.MBB == SwitchMBB) {
12697 CaseCluster &
Small = *
W.FirstCluster;
12698 CaseCluster &
Big = *
W.LastCluster;
12702 const APInt &SmallValue =
Small.Low->getValue();
12703 const APInt &BigValue =
Big.Low->getValue();
12706 APInt CommonBit = BigValue ^ SmallValue;
12713 DAG.getConstant(CommonBit,
DL, VT));
12715 DL, MVT::i1,
Or,
DAG.getConstant(BigValue | SmallValue,
DL, VT),
12721 addSuccessorWithProb(SwitchMBB,
Small.MBB,
Small.Prob +
Big.Prob);
12723 addSuccessorWithProb(
12724 SwitchMBB, DefaultMBB,
12728 addSuccessorWithProb(SwitchMBB, DefaultMBB);
12736 DAG.getBasicBlock(DefaultMBB));
12738 DAG.setRoot(BrCond);
12750 [](
const CaseCluster &a,
const CaseCluster &b) {
12751 return a.Prob != b.Prob ?
12753 a.Low->getValue().slt(b.Low->getValue());
12760 if (
I->Prob >
W.LastCluster->Prob)
12762 if (
I->Kind ==
CC_Range &&
I->MBB == NextMBB) {
12770 BranchProbability DefaultProb =
W.DefaultProb;
12771 BranchProbability UnhandledProbs = DefaultProb;
12773 UnhandledProbs +=
I->Prob;
12775 MachineBasicBlock *CurMBB =
W.MBB;
12777 bool FallthroughUnreachable =
false;
12778 MachineBasicBlock *Fallthrough;
12779 if (
I ==
W.LastCluster) {
12781 Fallthrough = DefaultMBB;
12782 FallthroughUnreachable =
12786 CurMF->
insert(BBI, Fallthrough);
12790 UnhandledProbs -=
I->Prob;
12795 JumpTableHeader *JTH = &
SL->JTCases[
I->JTCasesIndex].first;
12796 SwitchCG::JumpTable *JT = &
SL->JTCases[
I->JTCasesIndex].second;
12799 MachineBasicBlock *JumpMBB = JT->
MBB;
12800 CurMF->
insert(BBI, JumpMBB);
12802 auto JumpProb =
I->Prob;
12803 auto FallthroughProb = UnhandledProbs;
12811 if (*SI == DefaultMBB) {
12812 JumpProb += DefaultProb / 2;
12813 FallthroughProb -= DefaultProb / 2;
12832 if (FallthroughUnreachable) {
12834 bool HasCFBranch =
false;
12837 HasCFBranch = !CFBranch->isZero();
12838 if (!HasCFBranch &&
12844 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
12845 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
12854 if (CurMBB == SwitchMBB) {
12862 BitTestBlock *BTB = &
SL->BitTestCases[
I->BTCasesIndex];
12865 for (BitTestCase &BTC : BTB->
Cases)
12877 BTB->
Prob += DefaultProb / 2;
12881 if (FallthroughUnreachable)
12885 if (CurMBB == SwitchMBB) {
12892 const Value *
RHS, *
LHS, *MHS;
12894 if (
I->Low ==
I->High) {
12909 if (FallthroughUnreachable)
12913 CaseBlock CB(
CC,
LHS,
RHS, MHS,
I->MBB, Fallthrough, CurMBB,
12916 if (CurMBB == SwitchMBB)
12919 SL->SwitchCases.push_back(CB);
12924 CurMBB = Fallthrough;
12928void SelectionDAGBuilder::splitWorkItem(
SwitchWorkList &WorkList,
12929 const SwitchWorkListItem &W,
12932 assert(
W.FirstCluster->Low->getValue().slt(
W.LastCluster->Low->getValue()) &&
12933 "Clusters not sorted?");
12934 assert(
W.LastCluster -
W.FirstCluster + 1 >= 2 &&
"Too small to split!");
12936 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12937 SL->computeSplitWorkItemInfo(W);
12942 assert(PivotCluster >
W.FirstCluster);
12943 assert(PivotCluster <=
W.LastCluster);
12948 const ConstantInt *Pivot = PivotCluster->Low;
12957 MachineBasicBlock *LeftMBB;
12958 if (FirstLeft == LastLeft && FirstLeft->Kind ==
CC_Range &&
12959 FirstLeft->Low ==
W.GE &&
12960 (FirstLeft->High->getValue() + 1LL) == Pivot->
getValue()) {
12961 LeftMBB = FirstLeft->MBB;
12963 LeftMBB =
FuncInfo.MF->CreateMachineBasicBlock(
W.MBB->getBasicBlock());
12964 FuncInfo.MF->insert(BBI, LeftMBB);
12966 {LeftMBB, FirstLeft, LastLeft,
W.GE, Pivot,
W.DefaultProb / 2});
12974 MachineBasicBlock *RightMBB;
12975 if (FirstRight == LastRight && FirstRight->Kind ==
CC_Range &&
12976 W.LT && (FirstRight->High->getValue() + 1ULL) ==
W.LT->getValue()) {
12977 RightMBB = FirstRight->MBB;
12979 RightMBB =
FuncInfo.MF->CreateMachineBasicBlock(
W.MBB->getBasicBlock());
12980 FuncInfo.MF->insert(BBI, RightMBB);
12982 {RightMBB, FirstRight, LastRight, Pivot,
W.LT,
W.DefaultProb / 2});
12988 CaseBlock CB(
ISD::SETLT,
Cond, Pivot,
nullptr, LeftMBB, RightMBB,
W.MBB,
12991 if (
W.MBB == SwitchMBB)
12994 SL->SwitchCases.push_back(CB);
13019 MachineBasicBlock *SwitchMBB =
FuncInfo.MBB;
13027 unsigned PeeledCaseIndex = 0;
13028 bool SwitchPeeled =
false;
13029 for (
unsigned Index = 0;
Index < Clusters.size(); ++
Index) {
13030 CaseCluster &
CC = Clusters[
Index];
13031 if (
CC.Prob < TopCaseProb)
13033 TopCaseProb =
CC.Prob;
13034 PeeledCaseIndex =
Index;
13035 SwitchPeeled =
true;
13040 LLVM_DEBUG(
dbgs() <<
"Peeled one top case in switch stmt, prob: "
13041 << TopCaseProb <<
"\n");
13046 MachineBasicBlock *PeeledSwitchMBB =
13048 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
13051 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
13052 SwitchWorkListItem
W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
13053 nullptr,
nullptr, TopCaseProb.
getCompl()};
13054 lowerWorkItem(W,
SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
13056 Clusters.erase(PeeledCaseIt);
13057 for (CaseCluster &
CC : Clusters) {
13059 dbgs() <<
"Scale the probablity for one cluster, before scaling: "
13060 <<
CC.Prob <<
"\n");
13064 PeeledCaseProb = TopCaseProb;
13065 return PeeledSwitchMBB;
13068void SelectionDAGBuilder::visitSwitch(
const SwitchInst &
SI) {
13070 BranchProbabilityInfo *BPI =
FuncInfo.BPI;
13072 Clusters.reserve(
SI.getNumCases());
13073 for (
auto I :
SI.cases()) {
13074 MachineBasicBlock *Succ =
FuncInfo.getMBB(
I.getCaseSuccessor());
13075 const ConstantInt *CaseVal =
I.getCaseValue();
13076 BranchProbability Prob =
13078 : BranchProbability(1,
SI.getNumCases() + 1);
13082 MachineBasicBlock *DefaultMBB =
FuncInfo.getMBB(
SI.getDefaultDest());
13091 MachineBasicBlock *PeeledSwitchMBB =
13092 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
13095 MachineBasicBlock *SwitchMBB =
FuncInfo.MBB;
13096 if (Clusters.empty()) {
13097 assert(PeeledSwitchMBB == SwitchMBB);
13099 if (DefaultMBB != NextBlock(SwitchMBB)) {
13106 SL->findJumpTables(Clusters, &SI,
getCurSDLoc(), DefaultMBB,
DAG.getPSI(),
13108 SL->findBitTestClusters(Clusters, &SI);
13111 dbgs() <<
"Case clusters: ";
13112 for (
const CaseCluster &
C : Clusters) {
13118 C.Low->getValue().print(
dbgs(),
true);
13119 if (
C.Low !=
C.High) {
13121 C.High->getValue().print(
dbgs(),
true);
13128 assert(!Clusters.empty());
13132 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
13136 DefaultMBB ==
FuncInfo.getMBB(
SI.getDefaultDest()))
13139 {PeeledSwitchMBB,
First,
Last,
nullptr,
nullptr, DefaultProb});
13141 while (!WorkList.
empty()) {
13143 unsigned NumClusters =
W.LastCluster -
W.FirstCluster + 1;
13148 splitWorkItem(WorkList, W,
SI.getCondition(), SwitchMBB);
13152 lowerWorkItem(W,
SI.getCondition(), SwitchMBB, DefaultMBB);
13156void SelectionDAGBuilder::visitStepVector(
const CallInst &
I) {
13157 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
13163void SelectionDAGBuilder::visitVectorReverse(
const CallInst &
I) {
13164 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
13169 assert(VT ==
V.getValueType() &&
"Malformed vector.reverse!");
13178 SmallVector<int, 8>
Mask;
13180 for (
unsigned i = 0; i != NumElts; ++i)
13181 Mask.push_back(NumElts - 1 - i);
13186void SelectionDAGBuilder::visitVectorDeinterleave(
const CallInst &
I,
13195 EVT OutVT = ValueVTs[0];
13199 for (
unsigned i = 0; i != Factor; ++i) {
13200 assert(ValueVTs[i] == OutVT &&
"Expected VTs to be the same");
13202 DAG.getVectorIdxConstant(OutNumElts * i,
DL));
13206 DAG.getVTList(ValueVTs), SubVecs);
13210void SelectionDAGBuilder::visitVectorInterleave(
const CallInst &
I,
13213 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
13218 for (
unsigned i = 0; i < Factor; ++i) {
13221 "Expected VTs to be the same");
13229 for (
unsigned i = 0; i < Factor; ++i)
13236void SelectionDAGBuilder::visitFreeze(
const FreezeInst &
I) {
13240 unsigned NumValues = ValueVTs.
size();
13241 if (NumValues == 0)
return;
13246 for (
unsigned i = 0; i != NumValues; ++i)
13254void SelectionDAGBuilder::visitVectorSplice(
const CallInst &
I) {
13255 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
13261 const bool IsLeft =
I.getIntrinsicID() == Intrinsic::vector_splice_left;
13279 SmallVector<int, 8>
Mask;
13280 for (
unsigned i = 0; i < NumElts; ++i)
13281 Mask.push_back(Idx + i);
13309 assert(
MI->getOpcode() == TargetOpcode::COPY &&
13310 "start of copy chain MUST be COPY");
13311 Reg =
MI->getOperand(1).getReg();
13314 assert(
Reg.isVirtual() &&
"expected COPY of virtual register");
13315 MI =
MRI.def_begin(
Reg)->getParent();
13318 if (
MI->getOpcode() == TargetOpcode::COPY) {
13319 assert(
Reg.isVirtual() &&
"expected COPY of virtual register");
13320 Reg =
MI->getOperand(1).getReg();
13321 assert(
Reg.isPhysical() &&
"expected COPY of physical register");
13324 assert(
MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
13325 "end of copy chain MUST be INLINEASM_BR");
13335void SelectionDAGBuilder::visitCallBrLandingPad(
const CallInst &
I) {
13341 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
13342 const TargetRegisterInfo *
TRI =
DAG.getSubtarget().getRegisterInfo();
13343 MachineRegisterInfo &
MRI =
DAG.getMachineFunction().getRegInfo();
13351 for (
auto &
T : TargetConstraints) {
13352 SDISelAsmOperandInfo OpInfo(
T);
13360 switch (OpInfo.ConstraintType) {
13371 FuncInfo.MBB->addLiveIn(OriginalDef);
13379 ResultVTs.
push_back(OpInfo.ConstraintVT);
13388 ResultVTs.
push_back(OpInfo.ConstraintVT);
13396 DAG.getVTList(ResultVTs), ResultValues);
static unsigned getIntrinsicID(const SDNode *N)
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This file implements the BitVector class.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
static Value * getCondition(Instruction *I)
const HexagonInstrInfo * TII
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
static void computeConstraintToUse(const TargetLowering *TLI, TargetLowering::AsmOperandInfo &OpInfo)
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isUndef(const MachineInstr &MI)
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static const Function * getCalledFunction(const Value *V)
This file provides utility analysis objects describing memory locations.
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
OptimizedStructLayoutField Field
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static void failForInvalidBundles(const CallBase &I, StringRef Name, ArrayRef< uint32_t > AllowedBundles)
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info, const CallBase &Call, SelectionDAGBuilder &Builder, const TargetLowering &TLI, SelectionDAG &DAG)
Prepare DAG-level operands.
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static const GlobalValue * getGlobalAddressDbgOperand(const Value *V, DIExpression *&Expr, unsigned OpIdx, const MachineFunction &MF)
If V is the address of a describable global, possibly displaced by a constant, return the global and ...
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
DenseMap< const Argument *, std::pair< const AllocaInst *, const StoreInst * > > ArgCopyElisionMapTy
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< Register, TypeSize > > &Regs, const SDValue &N)
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static bool determineConstraints(ConstraintDecisionInfo &Info, TargetLowering::AsmOperandInfoVector &TargetConstraints, const CallBase &Call, SelectionDAGBuilder &Builder, const TargetLowering &TLI, const TargetMachine &TM, SelectionDAG &DAG, const BasicBlock *EHPadBB)
DetermineConstraints - Find the constraints to use for inline asm operands.
static bool constructOperandInfo(ConstraintDecisionInfo &Info, TargetLowering::AsmOperandInfoVector &TargetConstraints, SelectionDAGBuilder &Builder, const TargetLowering &TLI, ExtraFlags &ExtraInfo)
Construct operand info objects.
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static void ensureMatchedVecRegParts(unsigned NumRegs, unsigned NumParts, MVT RegisterVT, MVT PartVT)
Emit a fatal error if the broken down registers don't match part type or count.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
static FPClassTest getNoFPClass(const Instruction &I)
static LLVM_ATTRIBUTE_ALWAYS_INLINE MVT::SimpleValueType getSimpleVT(const uint8_t *MatcherTable, size_t &MatcherIndex)
getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value, use GetVBR to decode it.
This file defines the SmallPtrSet class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
uint16_t RegSizeInBits(const MCRegisterInfo &MRI, MCRegister RegNo)
static const fltSemantics & IEEEsingle()
static LLVM_ABI Semantics SemanticsToEnum(const llvm::fltSemantics &Sem)
static LLVM_ABI const fltSemantics * getArbitraryFPSemantics(StringRef Format)
Returns the fltSemantics for a given arbitrary FP format string, or nullptr if invalid.
Class for arbitrary precision integers.
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
an instruction to allocate memory on the stack
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
LLVM_ABI std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
This class represents an incoming formal argument to a Function.
LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
@ USubCond
Subtract only if no unsigned overflow.
@ FMinimum
*p = minimum(old, v) minimum matches the behavior of llvm.minimum.
@ Min
*p = old <signed v ? old : v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ FMaximum
*p = maximum(old, v) maximum matches the behavior of llvm.maximum.
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMaximumNum
*p = maximumnum(old, v) maximumnum matches the behavior of llvm.maximumnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
@ FMinimumNum
*p = minimumnum(old, v) minimumnum matches the behavior of llvm.minimumnum.
This class holds the attributes for a particular argument, parameter, function, or return value.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
InstListType::const_iterator const_iterator
LLVM_ABI bool isEntryBlock() const
Return true if this is the entry block of the containing function.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
This class represents a no-op cast from one type to another.
The address of a basic block.
Analysis providing branch probability information.
LLVM_ABI BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
LLVM_ABI bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static constexpr BranchProbability getOne()
static uint32_t getDenominator()
static constexpr BranchProbability getUnknown()
static constexpr BranchProbability getZero()
uint32_t getNumerator() const
LLVM_ABI uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
CallingConv::ID getCallingConv() const
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Value * getCalledOperand() const
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
bool isConvergent() const
Determine if the invoke is convergent.
FunctionType * getFunctionType() const
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
LLVM_ABI bool isTailCall() const
Tests if this call site is marked as a tail call.
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Conditional Branch instruction.
Class for constant bytes.
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
A constant value that is initialized with an expression using other constant values.
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
A signed pointer, in the ptrauth sense.
uint64_t getZExtValue() const
Constant Vector Declarations.
This is an important base class in LLVM.
This is the common base class for constrained floating point intrinsics.
LLVM_ABI std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
LLVM_ABI unsigned getNonMetadataArgCount() const
LLVM_ABI bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static LLVM_ABI void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static LLVM_ABI DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static LLVM_ABI std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
LLVM_ABI uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static LLVM_ABI std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static LLVM_ABI const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static LLVM_ABI DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static LLVM_ABI DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
Base class for variables.
LLVM_ABI std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
A parsed version of the target data layout string in and methods for querying it.
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
LocationType getType() const
LLVM_ABI Value * getVariableLocationOp(unsigned OpIdx) const
DIExpression * getExpression() const
DILocalVariable * getVariable() const
LLVM_ABI iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
LLVM_ABI DILocation * getInlinedAt() const
iterator find(const_arg_type_t< KeyT > Val)
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT, true > const_iterator
void reserve(size_type NumEntries)
Grow the DenseMap so that it can contain at least NumEntries items before resizing again.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for inline asm reporting.
static constexpr ElementCount getFixed(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
constexpr bool isScalar() const
Exactly one element.
Lightweight error class with error context and mandatory checking.
Class representing an expression and its matching format.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
bool allowReassoc() const
Flag queries.
An instruction for ordering other memory operations.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
MachineBasicBlock * getMBB(const BasicBlock *BB) const
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
MachineBasicBlock * MBB
MBB - The current block.
Class to represent function types.
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasParamAttribute(unsigned ArgNo, Attribute::AttrKind Kind) const
check if an attributes is in the list of attributes.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Constant * getPersonalityFn() const
Get the personality function associated with this function.
AttributeList getAttributes() const
Return the attribute list for this Function.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Garbage collection metadata for a single function.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
void setMemConstraint(ConstraintCode C)
setMemConstraint - Augment an existing flag with the constraint code for a memory constraint.
This instruction inserts a struct field of array element value into an aggregate value.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
LLVM_ABI AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
@ OB_clang_arc_attachedcall
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
A helper class to return the specified delimiter string after the first invocation of operator String...
An instruction for reading from memory.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
static LocationSize upperBound(uint64_t Value)
bool usesPerInvokeEHLabels(ExceptionHandling EH) const
Returns true if the exception tables reference the per-invoke EH labels emitted around invokes.
LLVM_ABI MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
LLVM_ABI MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
const MCAsmInfo & getAsmInfo() const
unsigned getID() const
getID() - Return the register class ID number.
bool isAllocatable() const
isAllocatable - Return true if this register class may be used to create virtual registers.
const MCPhysReg * iterator
iterator begin() const
begin/end - Return all of the registers in this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
const MDOperand & getOperand(unsigned I) const
LLVM_ABI StringRef getString() const
@ INVALID_SIMPLE_VALUE_TYPE
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
std::string getString() const
Return the value type as a string, e.g. i32.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
succ_iterator succ_begin()
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
SmallVectorImpl< MachineBasicBlock * >::iterator succ_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHContTarget(bool V=true)
Indicates if this is a target of Windows EH Continuation Guard.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
MachineInstrBundleIterator< MachineInstr > iterator
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
void setIsAliasedObjectIndex(int ObjectIdx, bool IsAliased)
Set "maybe pointed to by an LLVM IR value" for an object.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
bool hasEHFunclets() const
void setHasEHContTarget(bool V)
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI MCRegister getLiveInPhysReg(Register VReg) const
getLiveInPhysReg - If VReg is a live-in virtual register, return the corresponding live-in physical r...
An SDNode that represents everything that will be needed to construct a MachineInstr.
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
bool contains(const KeyT &Key) const
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
A Module instance is used to store all the information related to an LLVM module.
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
static PointerType * getUnqual(LLVMContext &C)
This constructs an opaque pointer to an object in the default address space (address space zero).
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(Register VReg)
static SDDbgOperand fromConst(const Value *Const)
static SDDbgOperand fromGlobalAddr(const GlobalValue *GV)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const CondBrInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
DenseMap< const Constant *, Register > ConstantsOut
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
SDValue lowerStartEH(SDValue Chain, const BasicBlock *EHPadBB, MCSymbol *&BeginLabel)
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr, const TargetLowering::PtrAuthInfo *PAI=nullptr)
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
const TargetTransformInfo * TTI
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
SDValue lowerNoFPClassToAssertNoFPClass(SelectionDAG &DAG, const Instruction &I, SDValue Op)
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void setValueToPoison(const Value *V, const SDLoc &dl)
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, Register Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
bool canTailCall(const CallBase &CB) const
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void CopyValueToVirtualRegister(const Value *V, Register Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
void init(GCFunctionInfo *gfi, BatchAAResults *BatchAA, AssumptionCache *AC, const TargetLibraryInfo *li, const TargetTransformInfo &TTI)
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
DebugLoc getCurDebugLoc() const
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
SDLoc getCurSDLoc() const
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
SDValue getFPOperationRoot(fp::ExceptionBehavior EB)
Return the current virtual root of the Selection DAG, flushing PendingConstrainedFP or PendingConstra...
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
void LowerCallSiteWithPtrAuthBundle(const CallBase &CB, const BasicBlock *EHPadBB)
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
void LowerDeoptimizingReturn()
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
MachineRegisterInfo * RegInfo
std::unique_ptr< SwiftErrorValueTracking > SwiftError
virtual void emitFunctionEntryCode()
std::unique_ptr< SelectionDAGBuilder > SDB
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemccpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue C, SDValue Size, const CallInst *CI) const
Emit target-specific code that performs a memccpy, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, const CallInst *CI) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrstr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, const CallInst *CI) const
Emit target-specific code that performs a strstr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo, const CallInst *CI) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, const CallInst *CI) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy, const CallInst *CI) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
LLVM_ABI bool shouldOptForSize() const
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
LLVM_ABI void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
LLVM_ABI SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
SDValue getTargetFrameIndex(int FI, EVT VT)
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
LLVM_ABI SDValue getBasicBlock(MachineBasicBlock *MBB)
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
LLVM_ABI SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
MachineBasicBlock * getParentMBB()
bool shouldEmitFunctionBasedCheckStackProtector() const
An instruction for storing to memory.
Represent a constant reference to a string, i.e.
constexpr bool empty() const
Check if the string is empty.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Provides information about what library functions are available for the current target.
virtual Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Returns the desired alignment for ByVal or InAlloca aggregate function arguments in the caller parame...
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
virtual bool isAtomicAlignmentSupported(Align Alignment, uint64_t SizeInBytes) const
Return true if the target supports an atomic access of SizeInBytes bytes at the given Alignment.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
virtual Register getExceptionSelectorRegister(ExceptionHandling EH, const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual bool useStackGuardMixFP() const
If this function returns true, stack protection checks should mix the frame pointer (or whichever poi...
MVT getRegisterType(LLVMContext &Context, EVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr, CodeGenOptLevel OptLevel=CodeGenOptLevel::Default) const
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
MachineMemOperand::Flags getVPIntrinsicMemOperandFlags(const VPIntrinsic &VPIntrin) const
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
virtual Value * getSDagStackGuard(const Module &M, const LibcallLoweringInfo &Libcalls) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
virtual Register getExceptionPointerRegister(ExceptionHandling EH, const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_UDIV, ISD::VP_SDIV,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual SDValue emitStackGuardMixFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue Ptr, SDValue Val, SDValue Mask) const
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
SDValue annotateStackObjectPointer(SDValue Ptr, SelectionDAG &DAG, const SDLoc &DL, Align Alignment) const
Annotate a stack object pointer with known-bits assertions.
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &, const Type *RetTy) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
Primary interface to the complete machine description for the target machine.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
CodeModel::Model getCodeModel() const
Returns the code model.
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM_ABI bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isTokenTy() const
Return true if this is 'token'.
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isVoidTy() const
Return true if this is 'void'.
Unconditional Branch instruction.
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
unsigned getNumOperands() const
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
This is the common base class for vector predication intrinsics.
static LLVM_ABI std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
LLVM_ABI MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
LLVMContext & getContext() const
All values hold a context through their type.
iterator_range< user_iterator > users()
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
const ParentTy * getParent() const
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ CONVERGENCECTRL_ANCHOR
The llvm.experimental.convergence.* intrinsics.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SET_FPENV
Sets the current floating-point environment.
@ ATOMIC_LOAD_FMINIMUMNUM
@ LOOP_DEPENDENCE_RAW_MASK
@ MASK_BEFOREFIRST
Has one mask vector operand.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ COND_LOOP
COND_LOOP is a conditional branch to self, used for implementing efficient conditional traps.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ STACKADDRESS
STACKADDRESS - Represents the llvm.stackaddress intrinsic.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ VECTOR_FIND_LAST_ACTIVE
Finds the index of the last active mask element Operands: Mask.
@ FMODF
FMODF - Decomposes the operand into integral and fractional parts, each having the same type and sign...
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ FSINCOSPI
FSINCOSPI - Compute both the sine and cosine times pi more accurately than FSINCOS(pi*x),...
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ VECTOR_SHUFFLE_VAR
VECTOR_SHUFFLE_VAR(VEC, MASK) - Returns VEC permuted by MASK, an integer vector with the same number ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ FMULADD
FMULADD - Performs a * b + c, with, or without, intermediate rounding.
@ FPTRUNC_ROUND
FPTRUNC_ROUND - This corresponds to the fptrunc_round intrinsic.
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ CLMUL
Carry-less multiplication operations.
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ CONVERT_FROM_ARBITRARY_FP
CONVERT_FROM_ARBITRARY_FP - This operator converts from an arbitrary floating-point represented as an...
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
@ SET_ROUNDING
Set rounding mode.
@ CONVERGENCECTRL_GLUE
This does not correspond to any convergence control intrinsic.
@ SIGN_EXTEND
Conversion operators.
@ PREALLOCATED_SETUP
PREALLOCATED_SETUP - This has 2 operands: an input chain and a SRCVALUE with the preallocated call Va...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ VECREDUCE_FMAXIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM nodes do not propagate NaNs and order signed zeroes using the llvm....
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SSUBO
Same for subtraction.
@ PREALLOCATED_ARG
PREALLOCATED_ARG - This has 3 operands: an input chain, a SRCVALUE with the preallocated call Value,...
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
@ PtrAuthGlobalAddress
A ptrauth constant.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ PATCHPOINT
The llvm.experimental.patchpoint.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ MASKED_UDIV
Masked vector arithmetic that returns poison on disabled lanes.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ PCMARKER
PCMARKER - This corresponds to the pcmarker intrinsic.
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ ATOMIC_LOAD_FMAXIMUMNUM
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ RELOC_NONE
Issue a no-op relocation against a given symbol at the current location.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ VECTOR_MATCH
VECTOR_MATCH - this corresponds to the llvm.experimental.vector.match intrinsic.
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ STACKMAP
The llvm.experimental.stackmap intrinsic.
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ CLEAR_CACHE
llvm.clear_cache intrinsic Operands: Input Chain, Start Addres, End Address Outputs: Output Chain
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ CONVERT_TO_ARBITRARY_FP
CONVERT_TO_ARBITRARY_FP - Converts a native FP value to an arbitrary floating-point format,...
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_REPEAT
VECTOR_REPEAT(FIXED_LENGTH_VECTOR) Repeatedly copies the elements of the source fixed-length vector t...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
@ LOOP_DEPENDENCE_WAR_MASK
The llvm.loop.dependence.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
Flag
These should be considered private to the implementation of the MCInstrDesc class.
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
auto m_VScale()
Matches a call to llvm.vscale().
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
LLVM_ABI void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
std::vector< CaseCluster > CaseClusterVector
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
CaseClusterVector::iterator CaseClusterIt
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
ExceptionBehavior
Exception behavior used for floating point operations.
@ ebStrict
This corresponds to "fpexcept.strict".
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
@ ebIgnore
This corresponds to "fpexcept.ignore".
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract_or_null(Y &&MD)
Extract a Value from Metadata, allowing null.
NodeAddr< FuncNode * > Func
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Type * getValueType(Value *V, bool ReVec, bool LookThroughCmp)
Returns the "element type" of the given value/instruction V.
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
LLVM_ABI bool canDescribeGlobalAddressInLocationList(const MachineFunction &MF)
Test if the debug info for MF can name a describable global address in a location list too,...
LLVM_ABI ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
LLVM_ABI cl::opt< bool > ProfcheckDisableMetadataFixes
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
SDValue peekThroughFreeze(SDValue V)
Return the non-frozen source operand of V if it exists.
LLVM_ABI void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
RelativeUniformCounterPtr Values
@ Unknown
Not known to have no common set bits.
LLVM_ABI void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs=nullptr, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
LLVM_ABI void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
LLVM_ABI bool isExceptionPointerAndSelectorType(Type *Ty)
Return true if landingpad result type Ty is a struct of an exception pointer (pointer or integer) and...
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
static ConstantRange getRange(Value *Op, SCCPSolver &Solver, const SmallPtrSetImpl< Value * > &InsertedValues)
Helper for getting ranges from Solver.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
auto cast_or_null(const Y &Val)
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
gep_type_iterator gep_type_end(const User *GEP)
LLVM_ABI LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
constexpr auto equal_to(T &&Arg)
Functor variant of std::equal_to that can be used as a UnaryPredicate in functional algorithms like a...
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
detail::concat_range< ValueT, RangeTs... > concat(RangeTs &&...Ranges)
Returns a concatenated range across two or more ranges.
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
LLVM_ABI void ComputeValueTypes(const DataLayout &DL, Type *Ty, SmallVectorImpl< Type * > &Types, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
Given an LLVM IR type, compute non-aggregate subtypes.
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
void sort(IteratorTy Start, IteratorTy End)
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
generic_gep_type_iterator<> gep_type_iterator
auto succ_size(const MachineBasicBlock *BB)
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
LLVM_ABI ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
LLVM_ABI MDNode * getMemCacheHint(const Instruction &I, unsigned OperandNo=0)
Return the hint payload for operand OperandNo of I, or nullptr if absent.
LLVM_ABI EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Sub
Subtraction of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
LLVM_ABI bool isOnlyUsedInZeroEqualityComparison(const Instruction *CtxI)
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
LLVM_ABI bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
DWARFExpression::Operation Op
@ Dynamic
Denotes mode unknown at compile time.
LLVM_ABI ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
LLVM_ABI const GlobalValue * getDescribableGlobalAddress(const Constant *C, int64_t &Offset, const MachineFunction &MF)
If C is the address of a global, possibly displaced by a constant, return that global and set Offset ...
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
LLVM_ABI std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
LLVM_ABI GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
LLVM_ABI Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
LLVM_ABI unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
@ End
Run the machine verifier once, at the end of the pipeline.
LLVM_ABI SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
MCRegisterClass TargetRegisterClass
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isRISCVVectorTuple() const
Return true if this is a vector value type.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
void setPointerAddrSpace(unsigned AS)
void setOrigAlign(Align A)
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
A lightweight accessor for an operand bundle meant to be passed around by value.
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< std::pair< Register, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
bool isABIMangled() const
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< Register, 4 > Regs
This list holds the registers assigned to the values.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
void setUnpredictable(bool b)
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
A MapVector that performs no allocations if smaller than a certain size.
MachineBasicBlock * Default
BranchProbability DefaultProb
MachineBasicBlock * Parent
bool FallthroughUnreachable
MachineBasicBlock * ThisBB
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
BranchProbability TrueProb
BranchProbability FalseProb
MachineBasicBlock * TrueBB
MachineBasicBlock * FalseBB
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
Register Reg
The virtual register containing the index of the jump table entry to jump to.
MachineBasicBlock * Default
The MBB of the default bb, which is a successor of the range check MBB.
unsigned JTI
The JumpTableIndex for this jump table in the function.
MachineBasicBlock * MBB
The MBB into which to emit the code for the indirect jump.
std::optional< SDLoc > SL
The debug location of the instruction this JumpTable was produced from.
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setDeactivationSymbol(GlobalValue *Sym)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
bool IsPostTypeLegalization
SmallVector< SDValue, 4 > InVals
Type * OrigRetTy
Original unlegalized return type.
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
Type * RetTy
Same as OrigRetTy, or partially legalized for soft float libcalls.
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setPtrAuth(PtrAuthInfo Value)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
This structure contains the information necessary for lowering pointer-authenticating indirect calls.
LLVM_ABI void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)