79#include "llvm/IR/IntrinsicsAArch64.h"
80#include "llvm/IR/IntrinsicsAMDGPU.h"
81#include "llvm/IR/IntrinsicsBPF.h"
82#include "llvm/IR/IntrinsicsWebAssembly.h"
118#define DEBUG_TYPE "isel"
126 cl::desc(
"Insert the experimental `assertalign` node."),
131 cl::desc(
"Generate low-precision inline sequences "
132 "for some float libcalls"),
138 cl::desc(
"Set the case probability threshold for peeling the case from a "
139 "switch statement. A value greater than 100 will void this "
162 std::optional<CallingConv::ID>
CC);
173 std::optional<CallingConv::ID>
CC = std::nullopt,
174 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
178 PartVT, ValueVT,
CC))
196 unsigned RoundBits = PartBits * RoundParts;
197 EVT RoundVT = RoundBits == ValueBits ?
203 if (RoundParts > 2) {
207 PartVT, HalfVT,
V, InChain);
220 unsigned OddParts =
NumParts - RoundParts;
223 OddVT,
V, InChain,
CC);
239 assert(ValueVT ==
EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
250 !PartVT.
isVector() &&
"Unexpected split");
262 if (PartEVT == ValueVT)
266 ValueVT.
bitsLT(PartEVT)) {
279 if (ValueVT.
bitsLT(PartEVT)) {
284 Val = DAG.
getNode(*AssertOp,
DL, PartEVT, Val,
299 llvm::Attribute::StrictFP)) {
301 DAG.
getVTList(ValueVT, MVT::Other), InChain, Val,
313 if (PartEVT == MVT::x86mmx && ValueVT.
isInteger() &&
314 ValueVT.
bitsLT(PartEVT)) {
323 const Twine &ErrMsg) {
326 return Ctx.emitError(ErrMsg);
329 if (CI->isInlineAsm()) {
331 *CI, ErrMsg +
", possible invalid constraint for vector type"));
334 return Ctx.emitError(
I, ErrMsg);
340 MVT RegisterVT,
MVT PartVT) {
341 if (NumRegs ==
NumParts && RegisterVT == PartVT)
345 Twine(
"Part count doesn't match vector breakdown! ") +
Twine(NumRegs) +
347 " RegisterVT, " + PartVT.
getString() +
" PartVT");
359 std::optional<CallingConv::ID> CallConv) {
362 const bool IsABIRegCopy = CallConv.has_value();
371 unsigned NumIntermediates;
375 NumRegs =
TLI.getVectorTypeBreakdownForCallingConv(
376 *DAG.
getContext(), *CallConv, ValueVT, IntermediateVT,
377 NumIntermediates, RegisterVT);
380 TLI.getVectorTypeBreakdown(*DAG.
getContext(), ValueVT, IntermediateVT,
381 NumIntermediates, RegisterVT);
387 "Part type sizes don't match!");
394 for (
unsigned i = 0; i !=
NumParts; ++i)
396 V, InChain, CallConv);
401 "Must expand into a divisible number of parts!");
402 unsigned Factor =
NumParts / NumIntermediates;
403 for (
unsigned i = 0; i != NumIntermediates; ++i)
405 IntermediateVT,
V, InChain, CallConv);
420 DL, BuiltVectorTy,
Ops);
426 if (PartEVT == ValueVT)
442 "Cannot narrow, it would be a lossy transformation");
448 if (PartEVT == ValueVT)
465 TLI.isTypeLegal(ValueVT))
473 }
else if (ValueVT.
bitsLT(PartEVT)) {
482 *DAG.
getContext(),
V,
"non-trivial scalar-to-vector conversion");
513 std::optional<CallingConv::ID> CallConv);
521 std::optional<CallingConv::ID> CallConv = std::nullopt,
525 if (
TLI.splitValueIntoRegisterParts(DAG,
DL, Val, Parts,
NumParts, PartVT,
537 "Copying to an illegal type!");
543 EVT PartEVT = PartVT;
544 if (PartEVT == ValueVT) {
565 "Unknown mismatch!");
568 if (PartVT == MVT::x86mmx)
579 "Unknown mismatch!");
582 if (PartVT == MVT::x86mmx)
589 "Failed to tile the value with PartVT!");
592 if (PartEVT != ValueVT) {
594 "scalar-to-vector conversion failed");
606 "Do not know what to expand to!");
608 unsigned RoundBits = RoundParts * PartBits;
609 unsigned OddParts =
NumParts - RoundParts;
618 std::reverse(Parts + RoundParts, Parts +
NumParts);
632 for (
unsigned StepSize =
NumParts; StepSize > 1; StepSize /= 2) {
633 for (
unsigned i = 0; i <
NumParts; i += StepSize) {
634 unsigned ThisBits = StepSize * PartBits / 2;
637 SDValue &Part1 = Parts[i+StepSize/2];
644 if (ThisBits == PartBits && ThisVT != PartVT) {
652 std::reverse(Parts, Parts + OrigNumParts);
674 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
676 "Cannot widen to illegal type");
680 }
else if (PartEVT != ValueEVT) {
695 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
706 std::optional<CallingConv::ID> CallConv) {
710 const bool IsABIRegCopy = CallConv.has_value();
713 EVT PartEVT = PartVT;
714 if (PartEVT == ValueVT) {
760 "lossy conversion of vector to scalar type");
775 unsigned NumIntermediates;
778 NumRegs =
TLI.getVectorTypeBreakdownForCallingConv(
779 *DAG.
getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
783 TLI.getVectorTypeBreakdown(*DAG.
getContext(), ValueVT, IntermediateVT,
784 NumIntermediates, RegisterVT);
789 "Mixing scalable and fixed vectors when copying in parts");
791 std::optional<ElementCount> DestEltCnt;
801 if (ValueVT == BuiltVectorTy) {
825 for (
unsigned i = 0; i != NumIntermediates; ++i) {
843 for (
unsigned i = 0; i !=
NumParts; ++i)
848 assert(NumIntermediates != 0 &&
"division by zero");
850 "Must expand into a divisible number of parts!");
851 unsigned Factor =
NumParts / NumIntermediates;
852 for (
unsigned i = 0; i != NumIntermediates; ++i)
860 if (
I.hasOperandBundlesOtherThan(AllowedBundles)) {
864 for (
unsigned i = 0, e =
I.getNumOperandBundles(); i != e; ++i) {
867 OS << LS << U.getTagName();
870 Twine(
"cannot lower ", Name)
876 EVT valuevt, std::optional<CallingConv::ID>
CC)
882 std::optional<CallingConv::ID>
CC) {
890 ?
TLI.getNumRegistersForCallingConv(Context, *
CC, ValueVT)
891 :
TLI.getNumRegisters(Context, ValueVT);
894 ?
TLI.getRegisterTypeForCallingConv(Context, *
CC, ValueVT)
895 :
TLI.getRegisterType(Context, ValueVT);
896 for (
unsigned i = 0; i != NumRegs; ++i)
897 Regs.push_back(Reg + i);
898 RegVTs.push_back(RegisterVT);
900 Reg = Reg.id() + NumRegs;
922 ?
TLI.getRegisterTypeForCallingConv(
927 for (
unsigned i = 0; i != NumRegs; ++i) {
933 *Glue =
P.getValue(2);
936 Chain =
P.getValue(1);
964 EVT FromVT(MVT::Other);
968 }
else if (NumSignBits > 1) {
976 assert(FromVT != MVT::Other);
982 RegisterVT, ValueVT, V, Chain,
CallConv);
998 unsigned NumRegs =
Regs.size();
1004 ?
TLI.getRegisterTypeForCallingConv(
1019 for (
unsigned i = 0; i != NumRegs; ++i) {
1031 if (NumRegs == 1 || Glue)
1042 Chain = Chains[NumRegs-1];
1048 unsigned MatchingIdx,
const SDLoc &dl,
1050 std::vector<SDValue> &
Ops)
const {
1055 Flag.setMatchingOp(MatchingIdx);
1056 else if (!
Regs.empty() &&
Regs.front().isVirtual()) {
1064 Flag.setRegClass(RC->
getID());
1075 "No 1:1 mapping from clobbers to regs?");
1076 Register SP =
TLI.getStackPointerRegisterToSaveRestore();
1078 for (
unsigned I = 0, E =
ValueVTs.size();
I != E; ++
I) {
1083 "If we clobbered the stack pointer, MFI should know about it.");
1092 for (
unsigned i = 0; i != NumRegs; ++i) {
1093 assert(Reg <
Regs.size() &&
"Mismatch in # registers expected");
1105 unsigned RegCount = std::get<0>(CountAndVT);
1106 MVT RegisterVT = std::get<1>(CountAndVT);
1124 SL->init(
DAG.getTargetLoweringInfo(), TM,
DAG.getDataLayout());
1125 CanDescribeGlobalAddressInLocationList =
1131 UnusedArgNodeMap.clear();
1133 PendingExports.clear();
1134 PendingConstrainedFP.clear();
1135 PendingConstrainedFPStrict.clear();
1143 DanglingDebugInfoMap.clear();
1150 if (Pending.
empty())
1156 unsigned i = 0, e = Pending.
size();
1157 for (; i != e; ++i) {
1159 if (Pending[i].
getNode()->getOperand(0) == Root)
1167 if (Pending.
size() == 1)
1194 if (!PendingConstrainedFPStrict.empty()) {
1195 assert(PendingConstrainedFP.empty());
1196 updateRoot(PendingConstrainedFPStrict);
1209 if (!PendingConstrainedFP.empty()) {
1210 assert(PendingConstrainedFPStrict.empty());
1211 updateRoot(PendingConstrainedFP);
1215 return DAG.getRoot();
1223 PendingConstrainedFP.size() +
1224 PendingConstrainedFPStrict.size());
1226 PendingConstrainedFP.end());
1227 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1228 PendingConstrainedFPStrict.end());
1229 PendingConstrainedFP.clear();
1230 PendingConstrainedFPStrict.clear();
1237 PendingExports.append(PendingConstrainedFPStrict.begin(),
1238 PendingConstrainedFPStrict.end());
1239 PendingConstrainedFPStrict.clear();
1240 return updateRoot(PendingExports);
1247 assert(Variable &&
"Missing variable");
1254 <<
"dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1270 if (IsParameter && FINode) {
1272 SDV =
DAG.getFrameIndexDbgValue(Variable,
Expression, FINode->getIndex(),
1273 true,
DL, SDNodeOrder);
1278 FuncArgumentDbgValueKind::Declare,
N);
1281 SDV =
DAG.getDbgValue(Variable,
Expression,
N.getNode(),
N.getResNo(),
1282 true,
DL, SDNodeOrder);
1284 DAG.AddDbgValue(SDV, IsParameter);
1289 FuncArgumentDbgValueKind::Declare,
N)) {
1291 <<
" (could not emit func-arg dbg_value)\n");
1302 for (
auto It = FnVarLocs->locs_begin(&
I),
End = FnVarLocs->locs_end(&
I);
1304 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1306 if (It->Values.isKillLocation(It->Expr)) {
1312 It->Values.hasArgList())) {
1315 FnVarLocs->getDILocalVariable(It->VariableID),
1316 It->Expr, Vals.
size() > 1, It->DL, SDNodeOrder);
1329 bool SkipDbgVariableRecords =
DAG.getFunctionVarLocs();
1332 for (
DbgRecord &DR :
I.getDbgRecordRange()) {
1334 assert(DLR->getLabel() &&
"Missing label");
1336 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);
1337 DAG.AddDbgLabel(SDV);
1341 if (SkipDbgVariableRecords)
1349 if (
FuncInfo.PreprocessedDVRDeclares.contains(&DVR))
1351 LLVM_DEBUG(
dbgs() <<
"SelectionDAG visiting dbg_declare: " << DVR
1377 SDNodeOrder, IsVariadic)) {
1388 if (
I.isTerminator()) {
1389 HandlePHINodesInSuccessorBlocks(
I.getParent());
1396 bool NodeInserted =
false;
1397 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1398 MDNode *PCSectionsMD =
I.getMetadata(LLVMContext::MD_pcsections);
1399 MDNode *MMRA =
I.getMetadata(LLVMContext::MD_mmra);
1400 if (PCSectionsMD || MMRA) {
1401 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1402 DAG, [&](
SDNode *) { NodeInserted =
true; });
1412 if (PCSectionsMD || MMRA) {
1413 auto It = NodeMap.find(&
I);
1414 if (It != NodeMap.end()) {
1416 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1418 DAG.addMMRAMetadata(It->second.getNode(), MMRA);
1419 }
else if (NodeInserted) {
1422 errs() <<
"warning: loosing !pcsections and/or !mmra metadata ["
1423 <<
I.getModule()->getName() <<
"]\n";
1432void SelectionDAGBuilder::visitPHI(
const PHINode &) {
1442#define HANDLE_INST(NUM, OPCODE, CLASS) \
1443 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1444#include "llvm/IR/Instruction.def"
1481 DanglingDebugInfoMap[
Values[0]].emplace_back(Var, Expr,
DL, Order);
1486 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1487 DIVariable *DanglingVariable = DDI.getVariable();
1489 if (DanglingVariable == Variable && Expr->
fragmentsOverlap(DanglingExpr)) {
1491 << printDDI(
nullptr, DDI) <<
"\n");
1497 for (
auto &DDIMI : DanglingDebugInfoMap) {
1498 DanglingDebugInfoVector &DDIV = DDIMI.second;
1502 for (
auto &DDI : DDIV)
1503 if (isMatchingDbgValue(DDI))
1506 erase_if(DDIV, isMatchingDbgValue);
1514 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1515 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1518 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1519 for (
auto &DDI : DDIV) {
1521 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1524 assert(Variable->isValidLocationForIntrinsic(
DL) &&
1525 "Expected inlined-at fields to agree");
1535 if (!EmitFuncArgumentDbgValue(V, Variable, Expr,
DL,
1536 FuncArgumentDbgValueKind::Value, Val)) {
1538 << printDDI(V, DDI) <<
"\n");
1545 <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to "
1546 << ValSDNodeOrder <<
"\n");
1547 SDV = getDbgValue(Val, Variable, Expr,
DL,
1548 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1549 DAG.AddDbgValue(SDV,
false);
1553 <<
" in EmitFuncArgumentDbgValue\n");
1555 LLVM_DEBUG(
dbgs() <<
"Dropping debug info for " << printDDI(V, DDI)
1559 DAG.getConstantDbgValue(Variable, Expr, Poison,
DL, DbgSDNodeOrder);
1560 DAG.AddDbgValue(SDV,
false);
1587 DanglingDebugInfo &DDI) {
1592 const Value *OrigV = V;
1596 unsigned SDOrder = DDI.getSDNodeOrder();
1605 auto HandleGlobalAddress = [&] {
1613 false,
DL, SDOrder,
false);
1614 DAG.AddDbgValue(SDV,
false);
1620 HandleGlobalAddress())
1642 if (!AdditionalValues.
empty())
1651 HandleGlobalAddress()) {
1653 dbgs() <<
"Salvaged debug location info for:\n " << *Var <<
"\n"
1654 << *OrigV <<
"\nBy stripping back to:\n " << *V <<
"\n");
1662 assert(OrigV &&
"V shouldn't be null");
1664 auto *SDV =
DAG.getConstantDbgValue(Var, Expr, Poison,
DL, SDOrder);
1665 DAG.AddDbgValue(SDV,
false);
1667 << printDDI(OrigV, DDI) <<
"\n");
1684 unsigned Order,
bool IsVariadic) {
1689 if (visitEntryValueDbgValue(
Values, Var, Expr, DbgLoc))
1704 if (CE->getOpcode() == Instruction::IntToPtr) {
1714 if (CanDescribeGlobalAddressInLocationList)
1716 V, Expr, OpIdx,
DAG.getMachineFunction())) {
1735 N = UnusedArgNodeMap[V];
1740 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1741 FuncArgumentDbgValueKind::Value,
N))
1768 bool IsParamOfFunc =
1776 auto VMI =
FuncInfo.ValueMap.find(V);
1777 if (VMI !=
FuncInfo.ValueMap.end()) {
1782 V->getType(), std::nullopt);
1788 unsigned BitsToDescribe = 0;
1790 BitsToDescribe = *VarSize;
1792 BitsToDescribe =
Fragment->SizeInBits;
1795 if (
Offset >= BitsToDescribe)
1798 unsigned RegisterSize = RegAndSize.second;
1799 unsigned FragmentSize = (
Offset + RegisterSize > BitsToDescribe)
1800 ? BitsToDescribe -
Offset
1803 Expr,
Offset, FragmentSize);
1807 Var, *FragmentExpr, RegAndSize.first,
false, DbgLoc, Order);
1808 DAG.AddDbgValue(SDV,
false);
1824 DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1825 false, DbgLoc, Order, IsVariadic);
1826 DAG.AddDbgValue(SDV,
false);
1832 for (
auto &Pair : DanglingDebugInfoMap)
1833 for (
auto &DDI : Pair.second)
1841 auto It =
FuncInfo.ValueMap.find(V);
1844 if (It !=
FuncInfo.ValueMap.end()) {
1848 DAG.getDataLayout(), InReg, Ty,
1865 if (
N.getNode())
return N;
1880 if (V->getType()->isVoidTy())
1885 V->getType(), ValueVTs);
1886 if (ValueVTs.
empty())
1920 EVT VT =
TLI.getValueType(
DAG.getDataLayout(), V->getType(),
true);
1937 return DAG.getSplatBuildVector(
1940 return DAG.getConstant(*CI,
DL, VT);
1952 getValue(CPA->getAddrDiscriminator()),
1953 getValue(CPA->getDiscriminator()));
1969 visit(CE->getOpcode(), *CE);
1971 assert(N1.
getNode() &&
"visit didn't populate the NodeMap!");
1977 for (
const Use &U :
C->operands()) {
1983 for (
unsigned i = 0, e = Val->
getNumValues(); i != e; ++i)
1984 Constants.push_back(
SDValue(Val, i));
1993 for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) {
1997 for (
unsigned i = 0, e = Val->
getNumValues(); i != e; ++i)
2006 if (
C->getType()->isStructTy() ||
C->getType()->isArrayTy()) {
2008 "Unknown struct or array constant!");
2012 unsigned NumElts = ValueVTs.
size();
2016 for (
unsigned i = 0; i != NumElts; ++i) {
2017 EVT EltVT = ValueVTs[i];
2019 Constants[i] =
DAG.getUNDEF(EltVT);
2030 return DAG.getBlockAddress(BA, VT);
2033 return getValue(Equiv->getGlobalValue());
2038 if (VT == MVT::aarch64svcount) {
2039 assert(
C->isNullValue() &&
"Can only zero this target type!");
2045 assert(
C->isNullValue() &&
"Can only zero this target type!");
2055 if (VT == MVT::externref || VT == MVT::funcref) {
2056 assert(
C->isNullValue() &&
"Can only zero this target type!");
2059 Intrinsic::ID IID = VT == MVT::externref ? Intrinsic::wasm_ref_null_extern
2060 : Intrinsic::wasm_ref_null_func;
2072 for (
unsigned i = 0; i != NumElements; ++i)
2099 return DAG.getFrameIndex(
2100 SI->second,
TLI.getValueType(
DAG.getDataLayout(), AI->getType()));
2107 Inst->getType(), std::nullopt);
2121void SelectionDAGBuilder::visitCatchPad(
const CatchPadInst &
I) {
2134 if (IsMSVCCXX || IsCoreCLR)
2140 MachineBasicBlock *TargetMBB =
FuncInfo.getMBB(
I.getSuccessor());
2141 FuncInfo.MBB->addSuccessor(TargetMBB);
2148 if (TargetMBB != NextBlock(
FuncInfo.MBB) ||
2157 DAG.getMachineFunction().setHasEHContTarget(
true);
2163 Value *ParentPad =
I.getCatchSwitchParentPad();
2166 SuccessorColor = &
FuncInfo.Fn->getEntryBlock();
2169 assert(SuccessorColor &&
"No parent funclet for catchret!");
2170 MachineBasicBlock *SuccessorColorMBB =
FuncInfo.getMBB(SuccessorColor);
2171 assert(SuccessorColorMBB &&
"No MBB for SuccessorColor!");
2176 DAG.getBasicBlock(SuccessorColorMBB));
2180void SelectionDAGBuilder::visitCleanupPad(
const CleanupPadInst &CPI) {
2186 FuncInfo.MBB->setIsEHFuncletEntry();
2187 FuncInfo.MBB->setIsCleanupFuncletEntry();
2217 UnwindDests.emplace_back(FuncInfo.
getMBB(EHPadBB), Prob);
2223 UnwindDests.emplace_back(FuncInfo.
getMBB(EHPadBB), Prob);
2224 UnwindDests.back().first->setIsEHScopeEntry();
2226 if (!IsWasmCXX && !IsWasmD)
2227 UnwindDests.back().first->setIsEHFuncletEntry();
2231 for (
const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2232 UnwindDests.emplace_back(FuncInfo.
getMBB(CatchPadBB), Prob);
2234 if (IsMSVCCXX || IsCoreCLR)
2235 UnwindDests.back().first->setIsEHFuncletEntry();
2237 UnwindDests.back().first->setIsEHScopeEntry();
2239 NewEHPadBB = CatchSwitch->getUnwindDest();
2245 if (BPI && NewEHPadBB)
2247 EHPadBB = NewEHPadBB;
2254 auto UnwindDest =
I.getUnwindDest();
2255 BranchProbabilityInfo *BPI =
FuncInfo.BPI;
2256 BranchProbability UnwindDestProb =
2261 for (
auto &UnwindDest : UnwindDests) {
2262 UnwindDest.first->setIsEHPad();
2263 addSuccessorWithProb(
FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2265 FuncInfo.MBB->normalizeSuccProbs();
2268 MachineBasicBlock *CleanupPadMBB =
2269 FuncInfo.getMBB(
I.getCleanupPad()->getParent());
2275void SelectionDAGBuilder::visitCatchSwitch(
const CatchSwitchInst &CSI) {
2279void SelectionDAGBuilder::visitRet(
const ReturnInst &
I) {
2280 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
2281 auto &
DL =
DAG.getDataLayout();
2293 if (
I.getParent()->getTerminatingDeoptimizeCall()) {
2307 Type *RetTy =
I.getOperand(0)->getType();
2308 Align BaseAlign =
DL.getPrefTypeAlign(RetTy);
2314 SmallVector<uint64_t, 4>
Offsets;
2316 unsigned NumValues = ValueVTs.
size();
2319 for (
unsigned i = 0; i != NumValues; ++i) {
2326 if (MemVTs[i] != ValueVTs[i])
2328 Chains[i] =
DAG.getStore(
2336 MVT::Other, Chains);
2337 }
else if (
I.getNumOperands() != 0) {
2340 unsigned NumValues =
Types.size();
2344 const Function *
F =
I.getParent()->getParent();
2347 I.getOperand(0)->getType(),
F->getCallingConv(),
2351 if (
F->getAttributes().hasRetAttr(Attribute::SExt))
2353 else if (
F->getAttributes().hasRetAttr(Attribute::ZExt))
2356 LLVMContext &
Context =
F->getContext();
2357 bool RetInReg =
F->getAttributes().hasRetAttr(Attribute::InReg);
2359 for (
unsigned j = 0;
j != NumValues; ++
j) {
2375 ISD::ArgFlagsTy
Flags = ISD::ArgFlagsTy();
2379 if (
I.getOperand(0)->getType()->isPointerTy()) {
2381 Flags.setPointerAddrSpace(
2385 if (NeedsRegBlock) {
2386 Flags.setInConsecutiveRegs();
2387 if (j == NumValues - 1)
2388 Flags.setInConsecutiveRegsLast();
2396 else if (
F->getAttributes().hasRetAttr(Attribute::NoExt))
2399 for (
unsigned i = 0; i <
NumParts; ++i) {
2402 VT, Types[j], 0, 0));
2412 const Function *
F =
I.getParent()->getParent();
2414 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2416 ISD::ArgFlagsTy
Flags = ISD::ArgFlagsTy();
2417 Flags.setSwiftError();
2429 bool isVarArg =
DAG.getMachineFunction().getFunction().isVarArg();
2431 DAG.getMachineFunction().getFunction().getCallingConv();
2432 Chain =
DAG.getTargetLoweringInfo().LowerReturn(
2437 "LowerReturn didn't return a valid chain!");
2448 if (V->getType()->isEmptyTy())
2451 auto VMI =
FuncInfo.ValueMap.find(V);
2452 if (VMI !=
FuncInfo.ValueMap.end()) {
2454 "Unused value assigned virtual registers!");
2467 if (
FuncInfo.isExportedInst(V))
return;
2479 if (VI->getParent() == FromBB)
2505 const BasicBlock *SrcBB = Src->getBasicBlock();
2506 const BasicBlock *DstBB = Dst->getBasicBlock();
2510 auto SuccSize = std::max<uint32_t>(
succ_size(SrcBB), 1);
2520 Src->addSuccessorWithoutProb(Dst);
2523 Prob = getEdgeProbability(Src, Dst);
2524 Src->addSuccessor(Dst, Prob);
2530 return I->getParent() == BB;
2554 if (CurBB == SwitchBB ||
2560 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2565 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2567 if (FC->hasNoNaNs() ||
2575 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1),
nullptr,
2577 SL->SwitchCases.push_back(CB);
2586 SL->SwitchCases.push_back(CB);
2594 unsigned Depth = 0) {
2603 if (Necessary !=
nullptr) {
2606 if (Necessary->contains(
I))
2614 for (
unsigned OpIdx = 0,
E =
I->getNumOperands(); OpIdx <
E; ++OpIdx)
2634 if (BPI !=
nullptr) {
2642 std::optional<bool> Likely;
2645 else if (BPI->
isEdgeHot(
I.getParent(), IfFalse))
2649 if (
Opc == (*Likely ? Instruction::And : Instruction::Or))
2661 if (CostThresh <= 0)
2682 Value *BrCond =
I.getCondition();
2683 auto ShouldCountInsn = [&RhsDeps, &BrCond](
const Instruction *Ins) {
2684 for (
const auto *U : Ins->users()) {
2687 if (UIns != BrCond && !RhsDeps.
contains(UIns))
2700 for (
unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2702 for (
const auto &InsPair : RhsDeps) {
2703 if (!ShouldCountInsn(InsPair.first)) {
2704 ToDrop = InsPair.first;
2708 if (ToDrop ==
nullptr)
2710 RhsDeps.erase(ToDrop);
2713 for (
const auto &InsPair : RhsDeps) {
2718 CostOfIncluding +=
TTI->getInstructionCost(
2721 if (CostOfIncluding > CostThresh)
2747 const Value *BOpOp0, *BOpOp1;
2761 if (BOpc == Instruction::And)
2762 BOpc = Instruction::Or;
2763 else if (BOpc == Instruction::Or)
2764 BOpc = Instruction::And;
2770 bool BOpIsInOrAndTree = BOpc && BOpc ==
Opc && BOp->
hasOneUse();
2775 TProb, FProb, InvertCond);
2785 if (
Opc == Instruction::Or) {
2806 auto NewTrueProb = TProb / 2;
2807 auto NewFalseProb = TProb / 2 + FProb;
2810 NewFalseProb, InvertCond);
2817 Probs[1], InvertCond);
2819 assert(
Opc == Instruction::And &&
"Unknown merge op!");
2839 auto NewTrueProb = TProb + FProb / 2;
2840 auto NewFalseProb = FProb / 2;
2843 NewFalseProb, InvertCond);
2850 Probs[1], InvertCond);
2859 if (Cases.size() != 2)
return true;
2863 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2864 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2865 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2866 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2872 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2873 Cases[0].
CC == Cases[1].
CC &&
2876 if (Cases[0].
CC ==
ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2878 if (Cases[0].
CC ==
ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2885void SelectionDAGBuilder::visitUncondBr(
const UncondBrInst &
I) {
2895 if (Succ0MBB != NextBlock(BrMBB) ||
2904void SelectionDAGBuilder::visitCondBr(
const CondBrInst &
I) {
2905 MachineBasicBlock *BrMBB =
FuncInfo.MBB;
2907 MachineBasicBlock *Succ0MBB =
FuncInfo.getMBB(
I.getSuccessor(0));
2911 const Value *CondVal =
I.getCondition();
2912 MachineBasicBlock *Succ1MBB =
FuncInfo.getMBB(
I.getSuccessor(1));
2931 bool IsUnpredictable =
I.hasMetadata(LLVMContext::MD_unpredictable);
2933 if (!
DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2936 const Value *BOp0, *BOp1;
2939 Opcode = Instruction::And;
2941 Opcode = Instruction::Or;
2948 DAG.getTargetLoweringInfo().getJumpConditionMergingParams(
2949 Opcode, BOp0, BOp1,
FuncInfo.Fn))) {
2951 getEdgeProbability(BrMBB, Succ0MBB),
2952 getEdgeProbability(BrMBB, Succ1MBB),
2957 assert(
SL->SwitchCases[0].ThisBB == BrMBB &&
"Unexpected lowering!");
2961 for (
unsigned i = 1, e =
SL->SwitchCases.size(); i != e; ++i) {
2968 SL->SwitchCases.erase(
SL->SwitchCases.begin());
2974 for (
unsigned i = 1, e =
SL->SwitchCases.size(); i != e; ++i)
2975 FuncInfo.MF->erase(
SL->SwitchCases[i].ThisBB);
2977 SL->SwitchCases.clear();
2983 nullptr, Succ0MBB, Succ1MBB, BrMBB,
getCurSDLoc(),
3004 if (CB.
TrueBB != NextBlock(SwitchBB)) {
3011 auto &
TLI =
DAG.getTargetLoweringInfo();
3035 Cond =
DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.
CC);
3047 Cond =
DAG.getSetCC(dl, MVT::i1, CmpOp,
DAG.getConstant(
High, dl, VT),
3051 VT, CmpOp,
DAG.getConstant(
Low, dl, VT));
3052 Cond =
DAG.getSetCC(dl, MVT::i1, SUB,
3067 if (CB.
TrueBB == NextBlock(SwitchBB)) {
3083 BrCond =
DAG.getNode(
ISD::BR, dl, MVT::Other, BrCond,
3086 DAG.setRoot(BrCond);
3092 assert(JT.
SL &&
"Should set SDLoc for SelectionDAG!");
3093 assert(JT.
Reg &&
"Should lower JT Header first!");
3094 EVT PTy =
DAG.getTargetLoweringInfo().getJumpTableRegTy(
DAG.getDataLayout());
3098 Index.getValue(1),
Table, Index);
3099 DAG.setRoot(BrJumpTable);
3107 assert(JT.
SL &&
"Should set SDLoc for SelectionDAG!");
3114 DAG.getConstant(JTH.
First, dl, VT));
3123 DAG.getZExtOrTrunc(
Sub, dl,
TLI.getJumpTableRegTy(
DAG.getDataLayout()));
3129 JT.
Reg = JumpTableReg;
3136 dl,
TLI.getSetCCResultType(
DAG.getDataLayout(), *
DAG.getContext(),
3137 Sub.getValueType()),
3141 MVT::Other, CopyTo, CMP,
3145 if (JT.
MBB != NextBlock(SwitchBB))
3146 BrCond =
DAG.getNode(
ISD::BR, dl, MVT::Other, BrCond,
3147 DAG.getBasicBlock(JT.
MBB));
3149 DAG.setRoot(BrCond);
3152 if (JT.
MBB != NextBlock(SwitchBB))
3154 DAG.getBasicBlock(JT.
MBB)));
3156 DAG.setRoot(CopyTo);
3180 if (PtrTy != PtrMemTy)
3196 auto &
DL =
DAG.getDataLayout();
3197 EVT PtrTy =
TLI.getFrameIndexTy(
DL);
3198 EVT PtrMemTy =
TLI.getPointerMemTy(
DL,
DL.getAllocaAddrSpace());
3205 SDValue StackSlotPtr =
DAG.getFrameIndex(FI, PtrTy);
3212 PtrMemTy, dl,
DAG.getEntryNode(), StackSlotPtr,
3220 if (
TLI.useStackGuardMixFP())
3221 GuardVal =
TLI.emitStackGuardMixFP(
DAG, GuardVal, dl);
3229 TLI.getSSPStackGuardCheck(M,
DAG.getLibcalls());
3230 assert(GuardCheckFn &&
"Guard check function is null");
3241 Entry.IsInReg =
true;
3242 Args.push_back(Entry);
3248 getValue(GuardCheckFn), std::move(Args));
3250 std::pair<SDValue, SDValue> Result =
TLI.LowerCallTo(CLI);
3251 DAG.setRoot(Result.second);
3259 if (
TLI.useStackGuardMixFP()) {
3261 if (
const Value *IRGuard =
TLI.getSDagStackGuard(M,
DAG.getLibcalls())) {
3263 Guard =
DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3269 Guard =
DAG.getPOISON(PtrMemTy);
3273 if (
TLI.useLoadStackGuardNode(M)) {
3276 if (
const Value *IRGuard =
TLI.getSDagStackGuard(M,
DAG.getLibcalls())) {
3278 Guard =
DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3284 Guard =
DAG.getPOISON(PtrMemTy);
3327 auto *GuardCheckFn =
TLI.getSSPStackGuardCheck(M,
DAG.getLibcalls());
3330 auto &
DL =
DAG.getDataLayout();
3331 EVT PtrTy =
TLI.getFrameIndexTy(
DL);
3332 EVT PtrMemTy =
TLI.getPointerMemTy(
DL,
DL.getAllocaAddrSpace());
3338 SDValue StackSlotPtr =
DAG.getFrameIndex(FI, PtrTy);
3344 PtrMemTy, dl,
DAG.getEntryNode(), StackSlotPtr,
3348 if (
TLI.useStackGuardMixFP())
3349 GuardVal =
TLI.emitStackGuardMixFP(
DAG, GuardVal, dl);
3359 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3360 Entry.IsInReg =
true;
3361 Args.push_back(Entry);
3367 getValue(GuardCheckFn), std::move(Args));
3369 Chain =
TLI.LowerCallTo(CLI).second;
3373 Chain =
TLI.makeLibCall(
DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
3396 DAG.getNode(
ISD::SUB, dl, VT, SwitchOp,
DAG.getConstant(
B.First, dl, VT));
3400 bool UsePtrType =
false;
3401 if (!
TLI.isTypeLegal(VT)) {
3414 VT =
TLI.getPointerTy(
DAG.getDataLayout());
3424 if (!
B.FallthroughUnreachable)
3425 addSuccessorWithProb(SwitchBB,
B.Default,
B.DefaultProb);
3426 addSuccessorWithProb(SwitchBB,
MBB,
B.Prob);
3430 if (!
B.FallthroughUnreachable) {
3433 TLI.getSetCCResultType(
DAG.getDataLayout(), *
DAG.getContext(),
3439 DAG.getBasicBlock(
B.Default));
3443 if (
MBB != NextBlock(SwitchBB))
3461 if (PopCount == 1) {
3465 dl,
TLI.getSetCCResultType(
DAG.getDataLayout(), *
DAG.getContext(), VT),
3468 }
else if (PopCount == BB.
Range) {
3471 dl,
TLI.getSetCCResultType(
DAG.getDataLayout(), *
DAG.getContext(), VT),
3476 DAG.getConstant(1, dl, VT), ShiftOp);
3480 VT, SwitchVal,
DAG.getConstant(
B.Mask, dl, VT));
3482 dl,
TLI.getSetCCResultType(
DAG.getDataLayout(), *
DAG.getContext(), VT),
3487 addSuccessorWithProb(SwitchBB,
B.TargetBB,
B.ExtraProb);
3489 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3497 Cmp,
DAG.getBasicBlock(
B.TargetBB));
3500 if (NextMBB != NextBlock(SwitchBB))
3501 BrAnd =
DAG.getNode(
ISD::BR, dl, MVT::Other, BrAnd,
3502 DAG.getBasicBlock(NextMBB));
3507void SelectionDAGBuilder::visitInvoke(
const InvokeInst &
I) {
3525 const Value *Callee(
I.getCalledOperand());
3528 visitInlineAsm(
I, EHPadBB);
3533 case Intrinsic::donothing:
3535 case Intrinsic::seh_try_begin:
3536 case Intrinsic::seh_scope_begin:
3537 case Intrinsic::seh_try_end:
3538 case Intrinsic::seh_scope_end:
3544 case Intrinsic::experimental_patchpoint_void:
3545 case Intrinsic::experimental_patchpoint:
3546 visitPatchpoint(
I, EHPadBB);
3548 case Intrinsic::experimental_gc_statepoint:
3554 case Intrinsic::wasm_throw: {
3556 std::array<SDValue, 4>
Ops = {
3567 case Intrinsic::wasm_rethrow: {
3568 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
3569 std::array<SDValue, 2>
Ops = {
3578 case Intrinsic::bpf_unwind_exit: {
3579 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
3581 std::array<SDValue, 2>
Ops = {
3587 DAG.setRoot(lowerEndEH(Chain, &
I, EHPadBB, BeginLabel));
3591 }
else if (
I.hasDeoptState()) {
3612 BranchProbabilityInfo *BPI =
FuncInfo.BPI;
3613 BranchProbability EHPadBBProb =
3619 addSuccessorWithProb(InvokeMBB, Return);
3620 for (
auto &UnwindDest : UnwindDests) {
3621 UnwindDest.first->setIsEHPad();
3622 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3628 DAG.getBasicBlock(Return)));
3637void SelectionDAGBuilder::visitCallBrIntrinsic(
const CallBrInst &
I) {
3640 DAG.getTargetLoweringInfo().getTgtMemIntrinsic(
3641 Infos,
I,
DAG.getMachineFunction(),
I.getIntrinsicID());
3642 assert(Infos.
empty() &&
"Intrinsic touches memory");
3645 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(
I);
3648 getTargetIntrinsicOperands(
I, HasChain, OnlyLoad);
3649 SDVTList VTs = getTargetIntrinsicVTList(
I, HasChain);
3653 getTargetNonMemIntrinsicNode(*
I.getType(), HasChain,
Ops, VTs);
3654 Result = handleTargetIntrinsicRet(
I, HasChain, OnlyLoad, Result);
3659void SelectionDAGBuilder::visitCallBr(
const CallBrInst &
I) {
3660 MachineBasicBlock *CallBrMBB =
FuncInfo.MBB;
3662 if (
I.isInlineAsm()) {
3669 assert(!
I.hasOperandBundles() &&
3670 "Can't have operand bundles for intrinsics");
3671 visitCallBrIntrinsic(
I);
3676 SmallPtrSet<BasicBlock *, 8> Dests;
3677 Dests.
insert(
I.getDefaultDest());
3687 if (
I.isInlineAsm()) {
3688 for (BasicBlock *Dest :
I.getIndirectDests()) {
3690 Target->setIsInlineAsmBrIndirectTarget();
3696 Target->setLabelMustBeEmitted();
3698 if (Dests.
insert(Dest).second)
3707 DAG.getBasicBlock(Return)));
3710void SelectionDAGBuilder::visitResume(
const ResumeInst &RI) {
3711 llvm_unreachable(
"SelectionDAGBuilder shouldn't visit resume instructions!");
3714void SelectionDAGBuilder::visitLandingPad(
const LandingPadInst &LP) {
3716 "Call to landingpad not in landing pad!");
3720 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
3725 PersonalityFn) == 0)
3739 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
3741 "landingpad result type must be a struct of an exception pointer and "
3742 "an integer selector",
3750 assert(ValueVTs.
size() == 2 &&
"Only two-valued landingpads are supported");
3755 if (
FuncInfo.ExceptionPointerVirtReg) {
3756 Ops[0] =
DAG.getZExtOrTrunc(
3757 DAG.getCopyFromReg(
DAG.getEntryNode(), dl,
3764 Ops[1] =
DAG.getZExtOrTrunc(
3765 DAG.getCopyFromReg(
DAG.getEntryNode(), dl,
3772 DAG.getVTList(ValueVTs),
Ops);
3780 if (JTB.first.HeaderBB ==
First)
3781 JTB.first.HeaderBB =
Last;
3794 for (
unsigned i = 0, e =
I.getNumSuccessors(); i != e; ++i) {
3796 bool Inserted =
Done.insert(BB).second;
3801 addSuccessorWithProb(IndirectBrMBB, Succ);
3811 if (!
I.shouldLowerToTrap(
DAG.getTarget().Options.TrapUnreachable,
3812 DAG.getTarget().Options.NoTrapAfterNoreturn))
3818void SelectionDAGBuilder::visitUnary(
const User &
I,
unsigned Opcode) {
3829void SelectionDAGBuilder::visitBinary(
const User &
I,
unsigned Opcode) {
3832 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3833 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3836 Flags.setExact(ExactOp->isExact());
3838 Flags.setDisjoint(DisjointOp->isDisjoint());
3849void SelectionDAGBuilder::visitShift(
const User &
I,
unsigned Opcode) {
3853 EVT ShiftTy =
DAG.getTargetLoweringInfo().getShiftAmountTy(
3858 if (!
I.getType()->isVectorTy() && Op2.
getValueType() != ShiftTy) {
3860 "Unexpected shift type");
3870 if (
const OverflowingBinaryOperator *OFBinOp =
3872 nuw = OFBinOp->hasNoUnsignedWrap();
3873 nsw = OFBinOp->hasNoSignedWrap();
3875 if (
const PossiblyExactOperator *ExactOp =
3877 exact = ExactOp->isExact();
3880 Flags.setExact(exact);
3881 Flags.setNoSignedWrap(nsw);
3882 Flags.setNoUnsignedWrap(nuw);
3888void SelectionDAGBuilder::visitSDiv(
const User &
I) {
3899void SelectionDAGBuilder::visitICmp(
const ICmpInst &
I) {
3905 auto &
TLI =
DAG.getTargetLoweringInfo();
3918 Flags.setSameSign(
I.hasSameSign());
3920 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
3926void SelectionDAGBuilder::visitFCmp(
const FCmpInst &
I) {
3933 if (FPMO->hasNoNaNs() ||
3934 (
DAG.isKnownNeverNaN(Op1) &&
DAG.isKnownNeverNaN(Op2)))
3938 Flags.copyFMF(*FPMO);
3940 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
3950 return isa<SelectInst>(V);
3954void SelectionDAGBuilder::visitSelect(
const User &
I) {
3958 unsigned NumValues = ValueVTs.
size();
3959 if (NumValues == 0)
return;
3969 bool IsUnaryAbs =
false;
3970 bool Negate =
false;
3976 Flags.setUnpredictable(
3981 EVT VT = ValueVTs[0];
3982 LLVMContext &Ctx = *
DAG.getContext();
3983 auto &
TLI =
DAG.getTargetLoweringInfo();
3993 bool UseScalarMinMax = VT.
isVector() &&
4002 switch (SPR.Flavor) {
4011 switch (SPR.NaNBehavior) {
4018 Flags.setNoSignedZeros(
true);
4026 switch (SPR.NaNBehavior) {
4033 Flags.setNoSignedZeros(
true);
4069 for (
unsigned i = 0; i != NumValues; ++i) {
4078 for (
unsigned i = 0; i != NumValues; ++i) {
4092void SelectionDAGBuilder::visitTrunc(
const User &
I) {
4095 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4099 Flags.setNoSignedWrap(
Trunc->hasNoSignedWrap());
4100 Flags.setNoUnsignedWrap(
Trunc->hasNoUnsignedWrap());
4106void SelectionDAGBuilder::visitZExt(
const User &
I) {
4110 auto &
TLI =
DAG.getTargetLoweringInfo();
4115 Flags.setNonNeg(PNI->hasNonNeg());
4120 if (
Flags.hasNonNeg() &&
4129void SelectionDAGBuilder::visitSExt(
const User &
I) {
4133 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4138void SelectionDAGBuilder::visitFPTrunc(
const User &
I) {
4145 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4148 DAG.getTargetConstant(
4153void SelectionDAGBuilder::visitFPExt(
const User &
I) {
4156 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4164void SelectionDAGBuilder::visitFPToUI(
const User &
I) {
4167 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4172void SelectionDAGBuilder::visitFPToSI(
const User &
I) {
4175 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4180void SelectionDAGBuilder::visitUIToFP(
const User &
I) {
4183 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4192void SelectionDAGBuilder::visitSIToFP(
const User &
I) {
4195 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4203void SelectionDAGBuilder::visitPtrToAddr(
const User &
I) {
4206 const auto &
TLI =
DAG.getTargetLoweringInfo();
4214void SelectionDAGBuilder::visitPtrToInt(
const User &
I) {
4218 auto &
TLI =
DAG.getTargetLoweringInfo();
4219 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4228void SelectionDAGBuilder::visitIntToPtr(
const User &
I) {
4232 auto &
TLI =
DAG.getTargetLoweringInfo();
4240void SelectionDAGBuilder::visitBitCast(
const User &
I) {
4243 EVT DestVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
4248 if (DestVT !=
N.getValueType())
4256 setValue(&
I,
DAG.getConstant(
C->getValue(), dl, DestVT,
false,
4262void SelectionDAGBuilder::visitAddrSpaceCast(
const User &
I) {
4263 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4264 const Value *
SV =
I.getOperand(0);
4268 unsigned SrcAS =
SV->getType()->getPointerAddressSpace();
4269 unsigned DestAS =
I.getType()->getPointerAddressSpace();
4271 if (!TM.isNoopAddrSpaceCast(
DAG.getDataLayout(), SrcAS, DestAS)) {
4274 Flags.setNonNull(ASC->hasNonNull());
4281void SelectionDAGBuilder::visitInsertElement(
const User &
I) {
4282 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4289 InVec, InVal, InIdx));
4292void SelectionDAGBuilder::visitBitInsert(
const User &
I) {
4296 EVT BaseVT =
Base.getValueType();
4298 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4302 "bitinsert val wider than base should be rejected by verifier");
4308 Val =
DAG.getBitcast(ValVT, Val);
4319 DAG.getNode(
ISD::SHL, dl, BaseVT,
DAG.getConstant(InsertMask, dl, BaseVT),
4321 SDValue ClearMask =
DAG.getNOT(dl, ShiftedMask, BaseVT);
4324 SDValue ExtVal =
DAG.getZExtOrTrunc(Val, dl, BaseVT);
4326 DAG.getNode(
ISD::SHL, dl, BaseVT, ExtVal, LegalShiftAmount);
4331void SelectionDAGBuilder::visitBitExtract(
const User &
I) {
4334 EVT SrcVT = Src.getValueType();
4335 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4340 "bitextract result wider than source should be rejected by verifier");
4354 Result =
DAG.getBitcast(ResultVT, Result);
4363void SelectionDAGBuilder::visitExtractElement(
const User &
I) {
4364 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4373void SelectionDAGBuilder::visitShuffleVector(
const User &
I) {
4378 Mask = SVI->getShuffleMask();
4382 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4384 EVT SrcVT =
Src1.getValueType();
4390 DAG.getVectorIdxConstant(0,
DL));
4401 unsigned MaskNumElts =
Mask.size();
4403 if (SrcNumElts == MaskNumElts) {
4409 if (SrcNumElts < MaskNumElts) {
4413 if (MaskNumElts % SrcNumElts == 0) {
4417 unsigned NumConcat = MaskNumElts / SrcNumElts;
4418 bool IsConcat =
true;
4419 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4420 for (
unsigned i = 0; i != MaskNumElts; ++i) {
4426 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4427 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4428 ConcatSrcs[i / SrcNumElts] != (
int)(Idx / SrcNumElts))) {
4433 ConcatSrcs[i / SrcNumElts] =
Idx / SrcNumElts;
4440 for (
auto Src : ConcatSrcs) {
4453 unsigned PaddedMaskNumElts =
alignTo(MaskNumElts, SrcNumElts);
4454 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4470 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4471 for (
unsigned i = 0; i != MaskNumElts; ++i) {
4473 if (Idx >= (
int)SrcNumElts)
4474 Idx -= SrcNumElts - PaddedMaskNumElts;
4482 if (MaskNumElts != PaddedMaskNumElts)
4484 DAG.getVectorIdxConstant(0,
DL));
4490 assert(SrcNumElts > MaskNumElts);
4494 int StartIdx[2] = {-1, -1};
4495 bool CanExtract =
true;
4496 for (
int Idx : Mask) {
4501 if (Idx >= (
int)SrcNumElts) {
4509 int NewStartIdx =
alignDown(Idx, MaskNumElts);
4510 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4511 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4515 StartIdx[
Input] = NewStartIdx;
4518 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4524 for (
unsigned Input = 0;
Input < 2; ++
Input) {
4526 if (StartIdx[Input] < 0)
4527 Src =
DAG.getUNDEF(VT);
4530 DAG.getVectorIdxConstant(StartIdx[Input],
DL));
4535 SmallVector<int, 8> MappedOps(Mask);
4536 for (
int &Idx : MappedOps) {
4537 if (Idx >= (
int)SrcNumElts)
4538 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4543 setValue(&
I,
DAG.getVectorShuffle(VT,
DL, Src1, Src2, MappedOps));
4552 for (
int Idx : Mask) {
4556 Res =
DAG.getUNDEF(EltVT);
4559 if (Idx >= (
int)SrcNumElts)
Idx -= SrcNumElts;
4562 DAG.getVectorIdxConstant(Idx,
DL));
4572 ArrayRef<unsigned> Indices =
I.getIndices();
4573 const Value *Op0 =
I.getOperand(0);
4575 Type *AggTy =
I.getType();
4582 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4588 unsigned NumAggValues = AggValueVTs.
size();
4589 unsigned NumValValues = ValValueVTs.
size();
4593 if (!NumAggValues) {
4601 for (; i != LinearIndex; ++i)
4602 Values[i] = IntoUndef ?
DAG.getUNDEF(AggValueVTs[i]) :
4607 for (; i != LinearIndex + NumValValues; ++i)
4608 Values[i] = FromUndef ?
DAG.getUNDEF(AggValueVTs[i]) :
4612 for (; i != NumAggValues; ++i)
4613 Values[i] = IntoUndef ?
DAG.getUNDEF(AggValueVTs[i]) :
4621 ArrayRef<unsigned> Indices =
I.getIndices();
4622 const Value *Op0 =
I.getOperand(0);
4624 Type *ValTy =
I.getType();
4629 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4633 unsigned NumValValues = ValValueVTs.
size();
4636 if (!NumValValues) {
4645 for (
unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4646 Values[i - LinearIndex] =
4655void SelectionDAGBuilder::visitGetElementPtr(
const User &
I) {
4656 Value *Op0 =
I.getOperand(0);
4662 auto &
TLI =
DAG.getTargetLoweringInfo();
4667 bool IsVectorGEP =
I.getType()->isVectorTy();
4668 ElementCount VectorElementCount =
4674 const Value *
Idx = GTI.getOperand();
4675 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4680 DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(
Field);
4690 N =
DAG.getMemBasePlusOffset(
4691 N,
DAG.getConstant(
Offset, dl,
N.getValueType()), dl, Flags);
4697 unsigned IdxSize =
DAG.getDataLayout().getIndexSizeInBits(AS);
4699 TypeSize ElementSize =
4700 GTI.getSequentialElementStride(
DAG.getDataLayout());
4705 bool ElementScalable = ElementSize.
isScalable();
4711 C =
C->getSplatValue();
4714 if (CI && CI->isZero())
4716 if (CI && !ElementScalable) {
4717 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4720 if (
N.getValueType().isVector())
4721 OffsVal =
DAG.getConstant(
4724 OffsVal =
DAG.getConstant(Offs, dl, IdxTy);
4731 Flags.setNoUnsignedWrap(
true);
4734 OffsVal =
DAG.getSExtOrTrunc(OffsVal, dl,
N.getValueType());
4736 N =
DAG.getMemBasePlusOffset(
N, OffsVal, dl, Flags);
4744 if (
N.getValueType().isVector()) {
4746 VectorElementCount);
4747 IdxN =
DAG.getSplat(VT, dl, IdxN);
4751 N =
DAG.getSplat(VT, dl,
N);
4757 IdxN =
DAG.getSExtOrTrunc(IdxN, dl,
N.getValueType());
4759 SDNodeFlags ScaleFlags;
4768 if (ElementScalable) {
4769 EVT VScaleTy =
N.getValueType().getScalarType();
4772 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4773 if (
N.getValueType().isVector())
4774 VScale =
DAG.getSplatVector(
N.getValueType(), dl, VScale);
4775 IdxN =
DAG.getNode(
ISD::MUL, dl,
N.getValueType(), IdxN, VScale,
4780 if (ElementMul != 1) {
4781 if (ElementMul.isPowerOf2()) {
4782 unsigned Amt = ElementMul.logBase2();
4785 DAG.getShiftAmountConstant(Amt,
N.getValueType(), dl),
4788 SDValue Scale =
DAG.getConstant(ElementMul.getZExtValue(), dl,
4790 IdxN =
DAG.getNode(
ISD::MUL, dl,
N.getValueType(), IdxN, Scale,
4800 SDNodeFlags AddFlags;
4804 N =
DAG.getMemBasePlusOffset(
N, IdxN, dl, AddFlags);
4808 if (IsVectorGEP && !
N.getValueType().isVector()) {
4810 N =
DAG.getSplat(VT, dl,
N);
4821 N =
DAG.getPtrExtendInReg(
N, dl, PtrMemTy);
4826void SelectionDAGBuilder::visitAlloca(
const AllocaInst &
I) {
4833 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4834 auto &
DL =
DAG.getDataLayout();
4835 TypeSize TySize =
I.getAllocationBaseSize(
DL);
4842 AllocSize =
DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4844 AllocSize =
DAG.getNode(
4846 DAG.getZExtOrTrunc(
DAG.getTypeSize(dl, MVT::i64, TySize), dl, IntPtr));
4851 Align StackAlign =
DAG.getSubtarget().getFrameLowering()->getStackAlign();
4852 if (*Alignment <= StackAlign)
4860 DAG.getConstant(StackAlignMask, dl, IntPtr),
4865 DAG.getSignedConstant(~StackAlignMask, dl, IntPtr));
4869 DAG.getConstant(Alignment ?
Alignment->value() : 0, dl, IntPtr)};
4879 return I.getMetadata(LLVMContext::MD_range);
4884 if (std::optional<ConstantRange> CR = CB->getRange())
4888 return std::nullopt;
4893 return CB->getRetNoFPClass();
4897void SelectionDAGBuilder::visitLoad(
const LoadInst &
I) {
4899 return visitAtomicLoad(
I);
4901 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
4902 const Value *
SV =
I.getOperand(0);
4907 if (Arg->hasSwiftErrorAttr())
4908 return visitLoadFromSwiftError(
I);
4912 if (Alloca->isSwiftError())
4913 return visitLoadFromSwiftError(
I);
4919 Type *Ty =
I.getType();
4923 unsigned NumValues = ValueVTs.
size();
4928 AAMDNodes AAInfo =
I.getAAMetadata();
4931 bool isVolatile =
I.isVolatile();
4936 bool ConstantMemory =
false;
4943 BatchAA->pointsToConstantMemory(MemoryLocation(
4948 Root =
DAG.getEntryNode();
4949 ConstantMemory =
true;
4953 Root =
DAG.getRoot();
4964 unsigned ChainI = 0;
4965 for (
unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4981 MachinePointerInfo PtrInfo =
4983 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4984 : MachinePointerInfo();
4986 SDValue A =
DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4988 DAG.getLoad(MemVTs[i], dl, Root,
A, PtrInfo, Alignment, MMOFlags,
4989 MMOMetadata(AAInfo, Ranges, MemCacheHint));
4990 Chains[ChainI] =
L.getValue(1);
4992 if (MemVTs[i] != ValueVTs[i])
4993 L =
DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);
4995 if (MDNode *NoFPClassMD =
I.getMetadata(LLVMContext::MD_nofpclass)) {
4996 uint64_t FPTestInt =
4998 cast<ConstantAsMetadata>(NoFPClassMD->getOperand(0))->getValue())
5000 if (FPTestInt != fcNone) {
5001 SDValue FPTestConst =
5002 DAG.getTargetConstant(FPTestInt, SDLoc(), MVT::i32);
5003 L = DAG.getNode(ISD::AssertNoFPClass, dl, L.getValueType(), L,
5010 if (!ConstantMemory) {
5016 PendingLoads.push_back(Chain);
5020 DAG.getVTList(ValueVTs),
Values));
5023void SelectionDAGBuilder::visitStoreToSwiftError(
const StoreInst &
I) {
5024 assert(
DAG.getTargetLoweringInfo().supportSwiftError() &&
5025 "call visitStoreToSwiftError when backend supports swifterror");
5028 SmallVector<uint64_t, 4>
Offsets;
5029 const Value *SrcV =
I.getOperand(0);
5031 SrcV->
getType(), ValueVTs,
nullptr, &Offsets, 0);
5032 assert(ValueVTs.
size() == 1 && Offsets[0] == 0 &&
5033 "expect a single EVT for swifterror");
5042 SDValue(Src.getNode(), Src.getResNo()));
5043 DAG.setRoot(CopyNode);
5046void SelectionDAGBuilder::visitLoadFromSwiftError(
const LoadInst &
I) {
5047 assert(
DAG.getTargetLoweringInfo().supportSwiftError() &&
5048 "call visitLoadFromSwiftError when backend supports swifterror");
5051 !
I.hasMetadata(LLVMContext::MD_nontemporal) &&
5052 !
I.hasMetadata(LLVMContext::MD_invariant_load) &&
5053 "Support volatile, non temporal, invariant for load_from_swift_error");
5055 const Value *
SV =
I.getOperand(0);
5056 Type *Ty =
I.getType();
5059 !
BatchAA->pointsToConstantMemory(MemoryLocation(
5061 I.getAAMetadata()))) &&
5062 "load_from_swift_error should not be constant memory");
5065 SmallVector<uint64_t, 4>
Offsets;
5067 ValueVTs,
nullptr, &Offsets, 0);
5068 assert(ValueVTs.
size() == 1 && Offsets[0] == 0 &&
5069 "expect a single EVT for swifterror");
5079void SelectionDAGBuilder::visitStore(
const StoreInst &
I) {
5081 return visitAtomicStore(
I);
5083 const Value *SrcV =
I.getOperand(0);
5084 const Value *PtrV =
I.getOperand(1);
5086 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5091 if (Arg->hasSwiftErrorAttr())
5092 return visitStoreToSwiftError(
I);
5096 if (Alloca->isSwiftError())
5097 return visitStoreToSwiftError(
I);
5104 SrcV->
getType(), ValueVTs, &MemVTs, &Offsets);
5105 unsigned NumValues = ValueVTs.
size();
5119 AAMDNodes AAInfo =
I.getAAMetadata();
5124 unsigned ChainI = 0;
5125 for (
unsigned i = 0; i != NumValues; ++i, ++ChainI) {
5135 MachinePointerInfo PtrInfo =
5137 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
5138 : MachinePointerInfo();
5142 if (MemVTs[i] != ValueVTs[i])
5143 Val =
DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
5145 DAG.getStore(Root, dl, Val,
Add, PtrInfo, Alignment, MMOFlags,
5146 MMOMetadata(AAInfo,
nullptr, MemCacheHint));
5147 Chains[ChainI] = St;
5153 DAG.setRoot(StoreNode);
5156void SelectionDAGBuilder::visitMaskedStore(
const CallInst &
I,
5157 bool IsCompressing) {
5160 Value *Src0Operand =
I.getArgOperand(0);
5161 Value *PtrOperand =
I.getArgOperand(1);
5162 Value *MaskOperand =
I.getArgOperand(2);
5170 EVT VT =
Src0.getValueType();
5172 const auto &
TLI =
DAG.getTargetLoweringInfo();
5176 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
5181 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
5182 MachinePointerInfo(PtrOperand), MMOFlags,
5184 MMOMetadata(
I.getAAMetadata(),
nullptr, MemCacheHint));
5187 !IsCompressing &&
TTI->hasConditionalLoadStoreForType(
5188 I.getArgOperand(0)->getType(),
true)
5194 DAG.setRoot(StoreNode);
5224 C =
C->getSplatValue();
5238 if (!
GEP ||
GEP->getParent() != CurBB)
5241 if (
GEP->getNumOperands() != 2)
5244 const Value *BasePtr =
GEP->getPointerOperand();
5245 const Value *IndexVal =
GEP->getOperand(
GEP->getNumOperands() - 1);
5251 TypeSize ScaleVal =
DL.getTypeAllocSize(
GEP->getResultElementType());
5256 if (ScaleVal != 1 &&
5268void SelectionDAGBuilder::visitMaskedScatter(
const CallInst &
I) {
5272 const Value *Ptr =
I.getArgOperand(1);
5275 EVT VT =
Src0.getValueType();
5277 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5286 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
5296 EVT IdxVT =
Index.getValueType();
5304 SDValue Scatter =
DAG.getMaskedScatter(
DAG.getVTList(MVT::Other), VT, sdl,
5306 DAG.setRoot(Scatter);
5310void SelectionDAGBuilder::visitMaskedLoad(
const CallInst &
I,
bool IsExpanding) {
5313 Value *PtrOperand =
I.getArgOperand(0);
5314 Value *MaskOperand =
I.getArgOperand(1);
5315 Value *Src0Operand =
I.getArgOperand(2);
5323 EVT VT =
Src0.getValueType();
5324 AAMDNodes AAInfo =
I.getAAMetadata();
5332 SDValue InChain = AddToChain ?
DAG.getRoot() :
DAG.getEntryNode();
5334 const auto &
TLI =
DAG.getTargetLoweringInfo();
5338 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
5340 if (
I.hasMetadata(LLVMContext::MD_invariant_load))
5343 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
5344 MachinePointerInfo(PtrOperand), MMOFlags,
5346 MMOMetadata(AAInfo, Ranges, MemCacheHint));
5353 TTI->hasConditionalLoadStoreForType(Src0Operand->
getType(),
5358 DAG.getMaskedLoad(VT, sdl, InChain, Ptr,
Offset, Mask, Src0, VT, MMO,
5365void SelectionDAGBuilder::visitSpeculativeLoad(
const CallInst &
I) {
5367 Value *PtrOperand =
I.getArgOperand(0);
5372 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5375 AAMDNodes AAInfo =
I.getAAMetadata();
5383 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
5385 if (
I.hasMetadata(LLVMContext::MD_invariant_load))
5388 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
5389 MachinePointerInfo(PtrOperand), MMOFlags,
5397void SelectionDAGBuilder::visitMaskedGather(
const CallInst &
I) {
5401 const Value *Ptr =
I.getArgOperand(0);
5405 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5418 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
5421 MMOMetadata(
I.getAAMetadata(), Ranges));
5430 EVT IdxVT =
Index.getValueType();
5439 DAG.getMaskedGather(
DAG.getVTList(VT, MVT::Other), VT, sdl,
Ops, MMO,
5455 SDVTList VTs =
DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5457 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5463 MachinePointerInfo(
I.getPointerOperand()), Flags, MemVT.
getStoreSize(),
5464 I.getAlign(), MMOMetadata(AAMDNodes(),
nullptr, MemCacheHint),
5465 SSID, SuccessOrdering, FailureOrdering);
5468 dl, MemVT, VTs, InChain,
5476 DAG.setRoot(OutChain);
5479void SelectionDAGBuilder::visitAtomicRMW(
const AtomicRMWInst &
I) {
5488 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5494 MachinePointerInfo(
I.getPointerOperand()), Flags, MemVT.
getStoreSize(),
5495 I.getAlign(), MMOMetadata(AAMDNodes(),
nullptr, MemCacheHint),
5499 DAG.getAtomic(NT, dl, MemVT, InChain,
5506 DAG.setRoot(OutChain);
5509void SelectionDAGBuilder::visitFence(
const FenceInst &
I) {
5511 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5514 Ops[1] =
DAG.getTargetConstant((
unsigned)
I.getOrdering(), dl,
5516 Ops[2] =
DAG.getTargetConstant(
I.getSyncScopeID(), dl,
5523void SelectionDAGBuilder::visitAtomicLoad(
const LoadInst &
I) {
5530 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5541 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
5542 MachinePointerInfo(
I.getPointerOperand()), Flags, MemVT.
getStoreSize(),
5543 I.getAlign(), MMOMetadata(AAMDNodes(), Ranges, MemCacheHint), SSID,
5554 L =
DAG.getPtrExtOrTrunc(L, dl, VT);
5557 DAG.setRoot(OutChain);
5560void SelectionDAGBuilder::visitAtomicStore(
const StoreInst &
I) {
5568 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5580 MachinePointerInfo(
I.getPointerOperand()), Flags, MemVT.
getStoreSize(),
5581 I.getAlign(), MMOMetadata(AAMDNodes(),
nullptr, MemCacheHint),
5586 Val =
DAG.getPtrExtOrTrunc(Val, dl, MemVT);
5593 DAG.setRoot(OutChain);
5601std::pair<bool, bool>
5602SelectionDAGBuilder::getTargetIntrinsicCallProperties(
const CallBase &
I) {
5604 bool HasChain = !
F->doesNotAccessMemory();
5606 HasChain &&
F->onlyReadsMemory() &&
F->willReturn() &&
F->doesNotThrow();
5608 return {HasChain, OnlyLoad};
5612 const CallBase &
I,
bool HasChain,
bool OnlyLoad,
5614 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5621 Ops.push_back(
DAG.getRoot());
5634 for (
unsigned i = 0, e =
I.arg_size(); i != e; ++i) {
5635 const Value *Arg =
I.getArgOperand(i);
5636 if (!
I.paramHasAttr(i, Attribute::ImmArg)) {
5644 assert(CI->getBitWidth() <= 64 &&
5645 "large intrinsic immediates not handled");
5646 Ops.push_back(
DAG.getTargetConstant(*CI, SDLoc(), VT));
5653 if (std::optional<OperandBundleUse> Bundle =
5655 auto *Sym = Bundle->Inputs[0].get();
5658 Ops.push_back(SDSym);
5661 if (std::optional<OperandBundleUse> Bundle =
5663 Value *Token = Bundle->Inputs[0].get();
5665 assert(
Ops.back().getValueType() != MVT::Glue &&
5666 "Did not expect another glue node here.");
5669 Ops.push_back(ConvControlToken);
5677 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5685 return DAG.getVTList(ValueVTs);
5689SDValue SelectionDAGBuilder::getTargetNonMemIntrinsicNode(
5712 if (
I.getType()->isVoidTy())
5727void SelectionDAGBuilder::visitTargetIntrinsic(
const CallInst &
I,
5729 auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(
I);
5732 if (!
DAG.getMachineFunction().getSubtarget().isIntrinsicSupported(
5735 DAG.getContext()->diagnose(DiagnosticInfoUnsupportedTargetIntrinsic(
5736 *
I.getFunction(), IntrinsicID,
DL.getDebugLoc()));
5741 if (HasChain && !OnlyLoad)
5750 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
5753 TargetLowering::IntrinsicInfo *
Info = !Infos.
empty() ? &Infos[0] :
nullptr;
5756 getTargetIntrinsicOperands(
I, HasChain, OnlyLoad, Info);
5757 SDVTList VTs = getTargetIntrinsicVTList(
I, HasChain);
5762 Flags.copyFMF(*FPMO);
5763 SelectionDAG::FlagInserter FlagsInserter(
DAG, Flags);
5770 if (!Infos.
empty()) {
5775 for (
const auto &Info : Infos) {
5778 MachinePointerInfo MPI;
5780 MPI = MachinePointerInfo(
Info.ptrVal,
Info.offset);
5781 else if (
Info.fallbackAddressSpace)
5782 MPI = MachinePointerInfo(*
Info.fallbackAddressSpace);
5783 EVT MemVT =
Info.memVT;
5785 if (
Size.hasValue() && !
Size.getValue())
5789 MPI,
Info.flags,
Size, Alignment,
I.getAAMetadata(),
Info.ssid,
5797 Result = getTargetNonMemIntrinsicNode(*
I.getType(), HasChain,
Ops, VTs);
5800 Result = handleTargetIntrinsicRet(
I, HasChain, OnlyLoad, Result);
5857 SDValue TwoToFractionalPartOfX;
5934 if (
Op.getValueType() == MVT::f32 &&
5958 if (
Op.getValueType() == MVT::f32 &&
6057 if (
Op.getValueType() == MVT::f32 &&
6141 return DAG.
getNode(
ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
6154 if (
Op.getValueType() == MVT::f32 &&
6231 return DAG.
getNode(
ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
6242 if (
Op.getValueType() == MVT::f32 &&
6255 bool IsExp10 =
false;
6256 if (
LHS.getValueType() == MVT::f32 &&
RHS.getValueType() == MVT::f32 &&
6260 IsExp10 = LHSC->isExactlyValue(Ten);
6287 unsigned Val = RHSC->getSExtValue();
6316 CurSquare, CurSquare);
6321 if (RHSC->getSExtValue() < 0)
6335 EVT VT =
LHS.getValueType();
6358 if ((ScaleInt > 0 || (Saturating &&
Signed)) &&
6359 (
TLI.isTypeLegal(VT) ||
6362 Opcode, VT, ScaleInt);
6397 switch (
N.getOpcode()) {
6401 Op.getValueType().getSizeInBits());
6426bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
6434 const TargetInstrInfo *
TII =
DAG.getSubtarget().getInstrInfo();
6438 auto MakeVRegDbgValue = [&](
Register Reg, DIExpression *FragExpr,
6443 auto &Inst =
TII->get(TargetOpcode::DBG_INSTR_REF);
6450 auto *NewDIExpr = FragExpr;
6457 return BuildMI(MF,
DL, Inst,
false, MOs, Variable, NewDIExpr);
6460 auto &Inst =
TII->get(TargetOpcode::DBG_VALUE);
6461 return BuildMI(MF,
DL, Inst, Indirect,
Reg, Variable, FragExpr);
6465 if (Kind == FuncArgumentDbgValueKind::Value) {
6470 if (!IsInEntryBlock)
6486 bool VariableIsFunctionInputArg =
Variable->isParameter() &&
6487 !
DL->getInlinedAt();
6489 if (!IsInPrologue && !VariableIsFunctionInputArg)
6523 if (VariableIsFunctionInputArg) {
6525 if (ArgNo >=
FuncInfo.DescribedArgs.size())
6526 FuncInfo.DescribedArgs.resize(ArgNo + 1,
false);
6527 else if (!IsInPrologue &&
FuncInfo.DescribedArgs.test(ArgNo))
6528 return !NodeMap[
V].getNode();
6533 bool IsIndirect =
false;
6534 std::optional<MachineOperand>
Op;
6536 int FI =
FuncInfo.getArgumentFrameIndex(Arg);
6537 if (FI != std::numeric_limits<int>::max())
6541 if (!
Op &&
N.getNode()) {
6544 if (ArgRegsAndSizes.
size() == 1)
6545 Reg = ArgRegsAndSizes.
front().first;
6548 MachineRegisterInfo &RegInfo = MF.
getRegInfo();
6555 IsIndirect =
Kind != FuncArgumentDbgValueKind::Value;
6559 if (!
Op &&
N.getNode()) {
6563 if (FrameIndexSDNode *FINode =
6570 auto splitMultiRegDbgValue =
6583 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6586 if (
Offset >= ExprFragmentSizeInBits)
6590 if (
Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6591 RegFragmentSizeInBits = ExprFragmentSizeInBits -
Offset;
6596 Expr,
Offset, RegFragmentSizeInBits);
6600 if (!FragmentExpr) {
6601 SDDbgValue *SDV =
DAG.getConstantDbgValue(
6603 DAG.AddDbgValue(SDV,
false);
6606 MachineInstr *NewMI = MakeVRegDbgValue(
6607 Reg, *FragmentExpr, Kind != FuncArgumentDbgValueKind::Value);
6608 FuncInfo.ArgDbgValues.push_back(NewMI);
6617 if (VMI !=
FuncInfo.ValueMap.end()) {
6618 const auto &
TLI =
DAG.getTargetLoweringInfo();
6619 RegsForValue RFV(
V->getContext(),
TLI,
DAG.getDataLayout(), VMI->second,
6620 V->getType(), std::nullopt);
6621 if (RFV.occupiesMultipleRegs())
6622 return splitMultiRegDbgValue(RFV.getRegsAndSizes());
6625 IsIndirect =
Kind != FuncArgumentDbgValueKind::Value;
6626 }
else if (ArgRegsAndSizes.
size() > 1) {
6629 return splitMultiRegDbgValue(ArgRegsAndSizes);
6637 "Expected inlined-at fields to agree");
6638 MachineInstr *NewMI =
nullptr;
6641 NewMI = MakeVRegDbgValue(
Op->getReg(), Expr, IsIndirect);
6643 NewMI =
BuildMI(MF,
DL,
TII->get(TargetOpcode::DBG_VALUE),
true, *
Op,
6647 FuncInfo.ArgDbgValues.push_back(NewMI);
6656 unsigned DbgSDNodeOrder) {
6668 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
6669 false, dl, DbgSDNodeOrder);
6671 return DAG.getDbgValue(Variable, Expr,
N.getNode(),
N.getResNo(),
6672 false, dl, DbgSDNodeOrder);
6677 case Intrinsic::smul_fix:
6679 case Intrinsic::umul_fix:
6681 case Intrinsic::smul_fix_sat:
6683 case Intrinsic::umul_fix_sat:
6685 case Intrinsic::sdiv_fix:
6687 case Intrinsic::udiv_fix:
6689 case Intrinsic::sdiv_fix_sat:
6691 case Intrinsic::udiv_fix_sat:
6704 "expected call_preallocated_setup Value");
6705 for (
const auto *U : PreallocatedSetup->
users()) {
6707 const Function *Fn = UseCall->getCalledFunction();
6708 if (!Fn || Fn->
getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6718bool SelectionDAGBuilder::visitEntryValueDbgValue(
6728 auto ArgIt =
FuncInfo.ValueMap.find(Arg);
6729 if (ArgIt ==
FuncInfo.ValueMap.end()) {
6731 dbgs() <<
"Dropping dbg.value: expression is entry_value but "
6732 "couldn't find an associated register for the Argument\n");
6735 Register ArgVReg = ArgIt->getSecond();
6737 for (
auto [PhysReg, VirtReg] :
FuncInfo.RegInfo->liveins())
6738 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6739 SDDbgValue *SDV =
DAG.getVRegDbgValue(
6740 Variable, Expr, PhysReg,
false , DbgLoc, SDNodeOrder);
6741 DAG.AddDbgValue(SDV,
false );
6744 LLVM_DEBUG(
dbgs() <<
"Dropping dbg.value: expression is entry_value but "
6745 "couldn't find a physical register\n");
6750void SelectionDAGBuilder::visitConvergenceControl(
const CallInst &
I,
6753 switch (Intrinsic) {
6754 case Intrinsic::experimental_convergence_anchor:
6757 case Intrinsic::experimental_convergence_entry:
6760 case Intrinsic::experimental_convergence_loop: {
6762 auto *Token = Bundle->Inputs[0].get();
6770void SelectionDAGBuilder::visitVectorHistogram(
const CallInst &
I,
6771 unsigned IntrinsicID) {
6774 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6775 "Tried to lower unsupported histogram type");
6781 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
6796 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
6797 MachinePointerInfo(AS),
6800 MMOMetadata(
I.getAAMetadata(), Ranges));
6809 EVT IdxVT =
Index.getValueType();
6820 SDValue ID =
DAG.getTargetConstant(IntrinsicID, sdl, MVT::i32);
6823 SDValue Histogram =
DAG.getMaskedHistogram(
DAG.getVTList(MVT::Other), VT, sdl,
6827 DAG.setRoot(Histogram);
6830void SelectionDAGBuilder::visitVectorExtractLastActive(
const CallInst &
I,
6832 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6833 "Tried lowering invalid vector extract last");
6835 const DataLayout &Layout =
DAG.getDataLayout();
6839 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
6849 EVT BoolVT =
Mask.getValueType().getScalarType();
6851 Result =
DAG.getSelect(sdl, ResVT, AnyActive, Result, PassThru);
6858void SelectionDAGBuilder::visitIntrinsicCall(
const CallInst &
I,
6860 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
6869 switch (Intrinsic) {
6872 visitTargetIntrinsic(
I, Intrinsic);
6874 case Intrinsic::vscale: {
6879 case Intrinsic::vastart: visitVAStart(
I);
return;
6880 case Intrinsic::vaend: visitVAEnd(
I);
return;
6881 case Intrinsic::vacopy: visitVACopy(
I);
return;
6882 case Intrinsic::returnaddress:
6887 case Intrinsic::addressofreturnaddress:
6892 case Intrinsic::sponentry:
6897 case Intrinsic::frameaddress:
6902 case Intrinsic::read_volatile_register:
6903 case Intrinsic::read_register: {
6904 Value *
Reg =
I.getArgOperand(0);
6910 DAG.getVTList(VT, MVT::Other), Chain,
RegName);
6915 case Intrinsic::write_register: {
6916 Value *
Reg =
I.getArgOperand(0);
6917 Value *RegValue =
I.getArgOperand(1);
6925 case Intrinsic::write_volatile_register: {
6926 Value *
Reg =
I.getArgOperand(0);
6927 Value *RegValue =
I.getArgOperand(1);
6944 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
6945 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(PhysReg);
6946 MVT RegVT = *TRI->legalclasstypes_begin(*RC);
6947 DAG.setRoot(DAG.getNode(ISD::FAKE_USE, sdl, MVT::Other,
6948 {WriteChain, DAG.getRegister(PhysReg, RegVT)}));
6950 DAG.setRoot(WriteChain);
6954 case Intrinsic::memcpy:
6955 case Intrinsic::memcpy_inline: {
6961 "memcpy_inline needs constant size");
6963 Align DstAlign = MCI.getDestAlign().valueOrOne();
6964 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6965 bool isVol = MCI.isVolatile();
6967 SDValue MC =
DAG.getMemcpy(Root, sdl, Dst, Src,
Size, DstAlign, SrcAlign,
6968 isVol, MCI.isForceInlined(), &
I, std::nullopt,
6969 MachinePointerInfo(
I.getArgOperand(0)),
6970 MachinePointerInfo(
I.getArgOperand(1)),
6972 updateDAGForMaybeTailCall(MC);
6975 case Intrinsic::memset:
6976 case Intrinsic::memset_inline: {
6982 "memset_inline needs constant size");
6984 Align DstAlign = MSII.getDestAlign().valueOrOne();
6985 bool isVol = MSII.isVolatile();
6988 Root, sdl, Dst, Value,
Size, DstAlign, isVol, MSII.isForceInlined(),
6989 &
I, MachinePointerInfo(
I.getArgOperand(0)),
I.getAAMetadata());
6990 updateDAGForMaybeTailCall(MC);
6993 case Intrinsic::memmove: {
6999 Align DstAlign = MMI.getDestAlign().valueOrOne();
7000 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
7001 bool isVol = MMI.isVolatile();
7004 Root, sdl, Op1, Op2, Op3, DstAlign, SrcAlign, isVol, &
I,
7006 MachinePointerInfo(
I.getArgOperand(0)),
7007 MachinePointerInfo(
I.getArgOperand(1)),
I.getAAMetadata(),
BatchAA);
7008 updateDAGForMaybeTailCall(MM);
7011 case Intrinsic::memcpy_element_unordered_atomic: {
7017 Type *LengthTy =
MI.getLength()->getType();
7018 unsigned ElemSz =
MI.getElementSizeInBytes();
7022 isTC, MachinePointerInfo(
MI.getRawDest()),
7023 MachinePointerInfo(
MI.getRawSource()));
7024 updateDAGForMaybeTailCall(MC);
7027 case Intrinsic::memmove_element_unordered_atomic: {
7033 Type *LengthTy =
MI.getLength()->getType();
7034 unsigned ElemSz =
MI.getElementSizeInBytes();
7038 isTC, MachinePointerInfo(
MI.getRawDest()),
7039 MachinePointerInfo(
MI.getRawSource()));
7040 updateDAGForMaybeTailCall(MC);
7043 case Intrinsic::memset_element_unordered_atomic: {
7049 Type *LengthTy =
MI.getLength()->getType();
7050 unsigned ElemSz =
MI.getElementSizeInBytes();
7054 isTC, MachinePointerInfo(
MI.getRawDest()));
7055 updateDAGForMaybeTailCall(MC);
7058 case Intrinsic::call_preallocated_setup: {
7060 SDValue SrcValue =
DAG.getSrcValue(PreallocatedCall);
7067 case Intrinsic::call_preallocated_arg: {
7069 SDValue SrcValue =
DAG.getSrcValue(PreallocatedCall);
7083 case Intrinsic::eh_typeid_for: {
7086 unsigned TypeID =
DAG.getMachineFunction().getTypeIDFor(GV);
7087 Res =
DAG.getConstant(
TypeID, sdl, MVT::i32);
7092 case Intrinsic::eh_return_i32:
7093 case Intrinsic::eh_return_i64:
7094 DAG.getMachineFunction().setCallsEHReturn(
true);
7101 case Intrinsic::eh_unwind_init:
7102 DAG.getMachineFunction().setCallsUnwindInit(
true);
7104 case Intrinsic::eh_dwarf_cfa:
7109 case Intrinsic::eh_sjlj_callsite: {
7111 assert(
FuncInfo.getCurrentCallSite() == 0 &&
"Overlapping call sites!");
7116 case Intrinsic::eh_sjlj_functioncontext: {
7118 MachineFrameInfo &MFI =
DAG.getMachineFunction().getFrameInfo();
7121 int FI =
FuncInfo.StaticAllocaMap[FnCtx];
7125 case Intrinsic::eh_sjlj_setjmp: {
7130 DAG.getVTList(MVT::i32, MVT::Other),
Ops);
7132 DAG.setRoot(
Op.getValue(1));
7135 case Intrinsic::eh_sjlj_longjmp:
7139 case Intrinsic::eh_sjlj_setup_dispatch:
7143 case Intrinsic::masked_gather:
7144 visitMaskedGather(
I);
7146 case Intrinsic::masked_load:
7149 case Intrinsic::masked_scatter:
7150 visitMaskedScatter(
I);
7152 case Intrinsic::masked_store:
7153 visitMaskedStore(
I);
7155 case Intrinsic::masked_expandload:
7156 visitMaskedLoad(
I,
true );
7158 case Intrinsic::masked_compressstore:
7159 visitMaskedStore(
I,
true );
7161 case Intrinsic::speculative_load:
7162 visitSpeculativeLoad(
I);
7164 case Intrinsic::powi:
7168 case Intrinsic::log:
7171 case Intrinsic::log2:
7175 case Intrinsic::log10:
7179 case Intrinsic::exp:
7182 case Intrinsic::exp2:
7186 case Intrinsic::pow:
7190 case Intrinsic::sqrt:
7191 case Intrinsic::fabs:
7192 case Intrinsic::sin:
7193 case Intrinsic::cos:
7194 case Intrinsic::tan:
7195 case Intrinsic::asin:
7196 case Intrinsic::acos:
7197 case Intrinsic::atan:
7198 case Intrinsic::sinh:
7199 case Intrinsic::cosh:
7200 case Intrinsic::tanh:
7201 case Intrinsic::exp10:
7202 case Intrinsic::floor:
7203 case Intrinsic::ceil:
7204 case Intrinsic::trunc:
7205 case Intrinsic::rint:
7206 case Intrinsic::nearbyint:
7207 case Intrinsic::round:
7208 case Intrinsic::roundeven:
7209 case Intrinsic::canonicalize: {
7212 switch (Intrinsic) {
7214 case Intrinsic::sqrt: Opcode =
ISD::FSQRT;
break;
7215 case Intrinsic::fabs: Opcode =
ISD::FABS;
break;
7216 case Intrinsic::sin: Opcode =
ISD::FSIN;
break;
7217 case Intrinsic::cos: Opcode =
ISD::FCOS;
break;
7218 case Intrinsic::tan: Opcode =
ISD::FTAN;
break;
7219 case Intrinsic::asin: Opcode =
ISD::FASIN;
break;
7220 case Intrinsic::acos: Opcode =
ISD::FACOS;
break;
7221 case Intrinsic::atan: Opcode =
ISD::FATAN;
break;
7222 case Intrinsic::sinh: Opcode =
ISD::FSINH;
break;
7223 case Intrinsic::cosh: Opcode =
ISD::FCOSH;
break;
7224 case Intrinsic::tanh: Opcode =
ISD::FTANH;
break;
7225 case Intrinsic::exp10: Opcode =
ISD::FEXP10;
break;
7226 case Intrinsic::floor: Opcode =
ISD::FFLOOR;
break;
7227 case Intrinsic::ceil: Opcode =
ISD::FCEIL;
break;
7228 case Intrinsic::trunc: Opcode =
ISD::FTRUNC;
break;
7229 case Intrinsic::rint: Opcode =
ISD::FRINT;
break;
7231 case Intrinsic::round: Opcode =
ISD::FROUND;
break;
7238 getValue(
I.getArgOperand(0)).getValueType(),
7242 case Intrinsic::atan2:
7244 getValue(
I.getArgOperand(0)).getValueType(),
7248 case Intrinsic::lround:
7249 case Intrinsic::llround:
7250 case Intrinsic::lrint:
7251 case Intrinsic::llrint: {
7254 switch (Intrinsic) {
7256 case Intrinsic::lround: Opcode =
ISD::LROUND;
break;
7258 case Intrinsic::lrint: Opcode =
ISD::LRINT;
break;
7259 case Intrinsic::llrint: Opcode =
ISD::LLRINT;
break;
7268 case Intrinsic::minnum:
7270 getValue(
I.getArgOperand(0)).getValueType(),
7274 case Intrinsic::maxnum:
7276 getValue(
I.getArgOperand(0)).getValueType(),
7280 case Intrinsic::minimum:
7282 getValue(
I.getArgOperand(0)).getValueType(),
7286 case Intrinsic::maximum:
7288 getValue(
I.getArgOperand(0)).getValueType(),
7292 case Intrinsic::minimumnum:
7294 getValue(
I.getArgOperand(0)).getValueType(),
7298 case Intrinsic::maximumnum:
7300 getValue(
I.getArgOperand(0)).getValueType(),
7304 case Intrinsic::copysign:
7306 getValue(
I.getArgOperand(0)).getValueType(),
7310 case Intrinsic::ldexp:
7312 getValue(
I.getArgOperand(0)).getValueType(),
7316 case Intrinsic::modf:
7317 case Intrinsic::sincos:
7318 case Intrinsic::sincospi:
7319 case Intrinsic::frexp: {
7321 switch (Intrinsic) {
7324 case Intrinsic::sincos:
7327 case Intrinsic::sincospi:
7330 case Intrinsic::modf:
7333 case Intrinsic::frexp:
7339 SDVTList VTs =
DAG.getVTList(ValueVTs);
7341 &
I,
DAG.getNode(Opcode, sdl, VTs,
getValue(
I.getArgOperand(0)), Flags));
7344 case Intrinsic::arithmetic_fence: {
7346 getValue(
I.getArgOperand(0)).getValueType(),
7350 case Intrinsic::fma:
7356#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
7357 case Intrinsic::INTRINSIC:
7358#include "llvm/IR/ConstrainedOps.def"
7361#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
7362#include "llvm/IR/VPIntrinsics.def"
7365 case Intrinsic::fptrunc_round: {
7369 std::optional<RoundingMode> RoundMode =
7377 SelectionDAG::FlagInserter FlagsInserter(
DAG, Flags);
7382 DAG.getTargetConstant((
int)*RoundMode, sdl, MVT::i32));
7387 case Intrinsic::fmuladd: {
7391 getValue(
I.getArgOperand(0)).getValueType(),
7398 getValue(
I.getArgOperand(0)).getValueType(),
7414 case Intrinsic::fptosi_sat: {
7421 case Intrinsic::fptoui_sat: {
7428 case Intrinsic::convert_from_arbitrary_fp: {
7433 const fltSemantics *SrcSem =
7436 DAG.getContext()->emitError(
7437 "convert_from_arbitrary_fp: not implemented format '" + FormatStr +
7448 DAG.getTargetConstant(
static_cast<int>(SemEnum), sdl, MVT::i32);
7453 case Intrinsic::convert_to_arbitrary_fp: {
7458 const fltSemantics *DstSem =
7461 DAG.getContext()->emitError(
7462 "convert_to_arbitrary_fp: not implemented format '" + FormatStr +
7474 "Dynamic rounding mode should have been rejected by the verifier");
7482 DAG.getTargetConstant(
static_cast<int>(SemEnum), sdl, MVT::i32);
7484 DAG.getTargetConstant(
static_cast<int>(*RoundMode), sdl, MVT::i32);
7485 SDValue SatConst =
DAG.getTargetConstant(Saturate, sdl, MVT::i32);
7487 SemConst, RoundConst, SatConst));
7490 case Intrinsic::set_rounding:
7496 case Intrinsic::is_fpclass: {
7497 const DataLayout DLayout =
DAG.getDataLayout();
7506 Flags.setNoFPExcept(
7507 !
F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
7523 case Intrinsic::get_fpenv: {
7524 const DataLayout DLayout =
DAG.getDataLayout();
7526 Align TempAlign =
DAG.getEVTAlign(EnvVT);
7541 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
7544 Chain =
DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7545 Res =
DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
7551 case Intrinsic::set_fpenv: {
7554 Align TempAlign =
DAG.getEVTAlign(EnvVT);
7567 Chain =
DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
7569 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
7572 Chain =
DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7577 case Intrinsic::reset_fpenv:
7580 case Intrinsic::get_fpmode:
7589 case Intrinsic::set_fpmode:
7594 case Intrinsic::reset_fpmode: {
7599 case Intrinsic::pcmarker: {
7604 case Intrinsic::readcyclecounter: {
7607 DAG.getVTList(MVT::i64, MVT::Other),
Op);
7612 case Intrinsic::readsteadycounter: {
7615 DAG.getVTList(MVT::i64, MVT::Other),
Op);
7620 case Intrinsic::bitreverse:
7622 getValue(
I.getArgOperand(0)).getValueType(),
7625 case Intrinsic::bswap:
7627 getValue(
I.getArgOperand(0)).getValueType(),
7630 case Intrinsic::cttz: {
7638 case Intrinsic::ctlz: {
7646 case Intrinsic::ctpop: {
7652 case Intrinsic::fshl:
7653 case Intrinsic::fshr: {
7654 bool IsFSHL =
Intrinsic == Intrinsic::fshl;
7658 EVT VT =
X.getValueType();
7669 case Intrinsic::clmul: {
7675 case Intrinsic::pext: {
7681 case Intrinsic::pdep: {
7687 case Intrinsic::smulh:
7688 case Intrinsic::umulh: {
7695 case Intrinsic::sadd_sat: {
7701 case Intrinsic::uadd_sat: {
7707 case Intrinsic::ssub_sat: {
7713 case Intrinsic::usub_sat: {
7719 case Intrinsic::sshl_sat:
7720 case Intrinsic::ushl_sat: {
7724 EVT ShiftTy =
DAG.getTargetLoweringInfo().getShiftAmountTy(
7729 if (!
I.getType()->isVectorTy() && Op2.
getValueType() != ShiftTy) {
7732 "Unexpected shift type");
7741 case Intrinsic::smul_fix:
7742 case Intrinsic::umul_fix:
7743 case Intrinsic::smul_fix_sat:
7744 case Intrinsic::umul_fix_sat: {
7752 case Intrinsic::sdiv_fix:
7753 case Intrinsic::udiv_fix:
7754 case Intrinsic::sdiv_fix_sat:
7755 case Intrinsic::udiv_fix_sat: {
7760 Op1, Op2, Op3,
DAG,
TLI));
7763 case Intrinsic::smax: {
7769 case Intrinsic::smin: {
7775 case Intrinsic::umax: {
7781 case Intrinsic::umin: {
7787 case Intrinsic::abs: {
7794 case Intrinsic::scmp: {
7801 case Intrinsic::ucmp: {
7808 case Intrinsic::stackaddress:
7809 case Intrinsic::stacksave: {
7814 Res =
DAG.getNode(SDOpcode, sdl,
DAG.getVTList(VT, MVT::Other),
Op);
7819 case Intrinsic::stackrestore:
7823 case Intrinsic::get_dynamic_area_offset: {
7832 case Intrinsic::stackguard: {
7839 Res =
DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);
7843 LLVMContext &Ctx = *
DAG.getContext();
7844 Ctx.
diagnose(DiagnosticInfoGeneric(
"unable to lower stackguard"));
7851 MachinePointerInfo(
Global, 0), Align,
7863 case Intrinsic::stackprotector: {
7885 Chain, sdl, Src, FIN,
7892 case Intrinsic::objectsize:
7895 case Intrinsic::is_constant:
7898 case Intrinsic::annotation:
7899 case Intrinsic::ptr_annotation:
7900 case Intrinsic::launder_invariant_group:
7905 case Intrinsic::type_test:
7906 case Intrinsic::public_type_test:
7907 case Intrinsic::type_checked_load:
7908 case Intrinsic::type_checked_load_relative: {
7913 DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
7916 " intrinsic must be lowered by the LowerTypeTests pass "
7917 "before code generation",
7926 case Intrinsic::assume:
7927 case Intrinsic::experimental_noalias_scope_decl:
7928 case Intrinsic::var_annotation:
7929 case Intrinsic::sideeffect:
7934 case Intrinsic::codeview_annotation: {
7945 case Intrinsic::init_trampoline: {
7953 Ops[4] =
DAG.getSrcValue(
I.getArgOperand(0));
7961 case Intrinsic::adjust_trampoline:
7966 case Intrinsic::gcroot: {
7967 assert(
DAG.getMachineFunction().getFunction().hasGC() &&
7968 "only valid in functions with gc specified, enforced by Verifier");
7970 const Value *Alloca =
I.getArgOperand(0)->stripPointerCasts();
7977 case Intrinsic::gcread:
7978 case Intrinsic::gcwrite:
7980 case Intrinsic::get_rounding:
7986 case Intrinsic::expect:
7987 case Intrinsic::expect_with_probability:
7993 case Intrinsic::ubsantrap:
7994 case Intrinsic::debugtrap:
7995 case Intrinsic::trap: {
7996 StringRef TrapFuncName =
7997 I.getAttributes().getFnAttr(
"trap-func-name").getValueAsString();
7998 if (TrapFuncName.
empty()) {
7999 switch (Intrinsic) {
8000 case Intrinsic::trap:
8003 case Intrinsic::debugtrap:
8006 case Intrinsic::ubsantrap:
8009 DAG.getTargetConstant(
8015 DAG.addNoMergeSiteInfo(
DAG.getRoot().getNode(),
8016 I.hasFnAttr(Attribute::NoMerge));
8020 if (Intrinsic == Intrinsic::ubsantrap) {
8021 Value *Arg =
I.getArgOperand(0);
8023 LibInfo->getExtAttrForI8Param(
false));
8026 TargetLowering::CallLoweringInfo CLI(
DAG);
8027 CLI.setDebugLoc(sdl).setChain(
getRoot()).setLibCallee(
8029 DAG.getExternalSymbol(TrapFuncName.
data(),
8032 CLI.NoMerge =
I.hasFnAttr(Attribute::NoMerge);
8038 case Intrinsic::allow_runtime_check:
8039 case Intrinsic::allow_ubsan_check:
8043 case Intrinsic::uadd_with_overflow:
8044 case Intrinsic::sadd_with_overflow:
8045 case Intrinsic::usub_with_overflow:
8046 case Intrinsic::ssub_with_overflow:
8047 case Intrinsic::umul_with_overflow:
8048 case Intrinsic::smul_with_overflow: {
8050 switch (Intrinsic) {
8052 case Intrinsic::uadd_with_overflow:
Op =
ISD::UADDO;
break;
8053 case Intrinsic::sadd_with_overflow:
Op =
ISD::SADDO;
break;
8054 case Intrinsic::usub_with_overflow:
Op =
ISD::USUBO;
break;
8055 case Intrinsic::ssub_with_overflow:
Op =
ISD::SSUBO;
break;
8056 case Intrinsic::umul_with_overflow:
Op =
ISD::UMULO;
break;
8057 case Intrinsic::smul_with_overflow:
Op =
ISD::SMULO;
break;
8065 SDVTList VTs =
DAG.getVTList(ResultVT, OverflowVT);
8069 case Intrinsic::prefetch: {
8084 std::nullopt, Flags);
8090 DAG.setRoot(Result);
8093 case Intrinsic::lifetime_start:
8094 case Intrinsic::lifetime_end: {
8095 bool IsStart = (
Intrinsic == Intrinsic::lifetime_start);
8101 if (!LifetimeObject)
8106 auto SI =
FuncInfo.StaticAllocaMap.find(LifetimeObject);
8107 if (SI ==
FuncInfo.StaticAllocaMap.end())
8111 Res =
DAG.getLifetimeNode(IsStart, sdl,
getRoot(), FrameIndex);
8115 case Intrinsic::pseudoprobe: {
8123 case Intrinsic::invariant_start:
8128 case Intrinsic::invariant_end:
8131 case Intrinsic::clear_cache: {
8136 {InputChain, StartVal, EndVal});
8141 case Intrinsic::donothing:
8142 case Intrinsic::seh_try_begin:
8143 case Intrinsic::seh_scope_begin:
8144 case Intrinsic::seh_try_end:
8145 case Intrinsic::seh_scope_end:
8148 case Intrinsic::experimental_stackmap:
8151 case Intrinsic::experimental_patchpoint_void:
8152 case Intrinsic::experimental_patchpoint:
8155 case Intrinsic::experimental_gc_statepoint:
8158 case Intrinsic::experimental_gc_result:
8161 case Intrinsic::experimental_gc_relocate:
8164 case Intrinsic::instrprof_cover:
8166 case Intrinsic::instrprof_increment:
8168 case Intrinsic::instrprof_timestamp:
8170 case Intrinsic::instrprof_value_profile:
8172 case Intrinsic::instrprof_mcdc_parameters:
8174 case Intrinsic::instrprof_mcdc_tvbitmap_update:
8176 case Intrinsic::localescape: {
8178 const TargetInstrInfo *
TII =
DAG.getSubtarget().getInstrInfo();
8182 for (
unsigned Idx = 0,
E =
I.arg_size(); Idx <
E; ++Idx) {
8188 "can only escape static allocas");
8193 TII->get(TargetOpcode::LOCAL_ESCAPE))
8201 case Intrinsic::localrecover: {
8209 unsigned(
Idx->getLimitedValue(std::numeric_limits<int>::max()));
8213 Value *
FP =
I.getArgOperand(1);
8219 SDValue OffsetSym =
DAG.getMCSymbol(FrameAllocSym, PtrVT);
8224 SDValue Add =
DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
8230 case Intrinsic::fake_use: {
8231 Value *
V =
I.getArgOperand(0);
8236 auto FakeUseValue = [&]() ->
SDValue {
8250 if (!FakeUseValue || FakeUseValue.isUndef())
8253 Ops[1] = FakeUseValue;
8262 case Intrinsic::reloc_none: {
8267 DAG.getTargetExternalSymbol(
8273 case Intrinsic::cond_loop: {
8283 case Intrinsic::eh_exceptionpointer:
8284 case Intrinsic::eh_exceptioncode: {
8290 SDValue N =
DAG.getCopyFromReg(
DAG.getEntryNode(), sdl, VReg, PtrVT);
8291 if (Intrinsic == Intrinsic::eh_exceptioncode)
8292 N =
DAG.getZExtOrTrunc(
N, sdl, MVT::i32);
8296 case Intrinsic::xray_customevent: {
8299 const auto &Triple =
DAG.getTarget().getTargetTriple();
8300 if (!Triple.isAArch64(64) && Triple.getArch() !=
Triple::x86_64 &&
8309 SDVTList NodeTys =
DAG.getVTList(MVT::Other, MVT::Glue);
8311 Ops.push_back(LogEntryVal);
8312 Ops.push_back(StrSizeVal);
8313 Ops.push_back(Chain);
8319 MachineSDNode *MN =
DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
8322 DAG.setRoot(patchableNode);
8326 case Intrinsic::xray_typedevent: {
8329 const auto &Triple =
DAG.getTarget().getTargetTriple();
8330 if (!Triple.isAArch64(64) && Triple.getArch() !=
Triple::x86_64 &&
8342 SDVTList NodeTys =
DAG.getVTList(MVT::Other, MVT::Glue);
8344 Ops.push_back(LogTypeId);
8345 Ops.push_back(LogEntryVal);
8346 Ops.push_back(StrSizeVal);
8347 Ops.push_back(Chain);
8353 MachineSDNode *MN =
DAG.getMachineNode(
8354 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys,
Ops);
8356 DAG.setRoot(patchableNode);
8360 case Intrinsic::experimental_deoptimize:
8363 case Intrinsic::stepvector:
8366 case Intrinsic::vector_reduce_fadd:
8367 case Intrinsic::vector_reduce_fmul:
8368 case Intrinsic::vector_reduce_add:
8369 case Intrinsic::vector_reduce_mul:
8370 case Intrinsic::vector_reduce_and:
8371 case Intrinsic::vector_reduce_or:
8372 case Intrinsic::vector_reduce_xor:
8373 case Intrinsic::vector_reduce_smax:
8374 case Intrinsic::vector_reduce_smin:
8375 case Intrinsic::vector_reduce_umax:
8376 case Intrinsic::vector_reduce_umin:
8377 case Intrinsic::vector_reduce_fmax:
8378 case Intrinsic::vector_reduce_fmin:
8379 case Intrinsic::vector_reduce_fmaximum:
8380 case Intrinsic::vector_reduce_fminimum:
8381 case Intrinsic::vector_reduce_fmaximumnum:
8382 case Intrinsic::vector_reduce_fminimumnum:
8383 visitVectorReduce(
I, Intrinsic);
8386 case Intrinsic::icall_branch_funnel: {
8392 I.getArgOperand(1),
Offset,
DAG.getDataLayout()));
8395 "llvm.icall.branch.funnel operand must be a GlobalValue");
8396 Ops.push_back(
DAG.getTargetGlobalAddress(
Base, sdl, MVT::i64, 0));
8398 struct BranchFunnelTarget {
8404 for (
unsigned Op = 1,
N =
I.arg_size();
Op !=
N;
Op += 2) {
8407 if (ElemBase !=
Base)
8409 "to the same GlobalValue");
8415 "llvm.icall.branch.funnel operand must be a GlobalValue");
8421 [](
const BranchFunnelTarget &
T1,
const BranchFunnelTarget &T2) {
8422 return T1.Offset < T2.Offset;
8425 for (
auto &
T : Targets) {
8426 Ops.push_back(
DAG.getTargetConstant(
T.Offset, sdl, MVT::i32));
8427 Ops.push_back(
T.Target);
8430 Ops.push_back(
DAG.getRoot());
8431 SDValue N(
DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
8440 case Intrinsic::wasm_landingpad_index:
8446 case Intrinsic::aarch64_settag:
8447 case Intrinsic::aarch64_settag_zero: {
8448 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
8449 bool ZeroMemory =
Intrinsic == Intrinsic::aarch64_settag_zero;
8452 getValue(
I.getArgOperand(1)), MachinePointerInfo(
I.getArgOperand(0)),
8458 case Intrinsic::amdgcn_cs_chain: {
8463 Type *RetTy =
I.getType();
8473 for (
unsigned Idx : {2, 3, 1}) {
8474 TargetLowering::ArgListEntry Arg(
getValue(
I.getOperand(Idx)),
8476 Arg.setAttributes(&
I, Idx);
8477 Args.push_back(Arg);
8480 assert(Args[0].IsInReg &&
"SGPR args should be marked inreg");
8481 assert(!Args[1].IsInReg &&
"VGPR args should not be marked inreg");
8482 Args[2].IsInReg =
true;
8485 for (
unsigned Idx = 4;
Idx <
I.arg_size(); ++
Idx) {
8486 TargetLowering::ArgListEntry Arg(
getValue(
I.getOperand(Idx)),
8488 Arg.setAttributes(&
I, Idx);
8489 Args.push_back(Arg);
8492 TargetLowering::CallLoweringInfo CLI(
DAG);
8495 .setCallee(
CC, RetTy, Callee, std::move(Args))
8498 .setConvergent(
I.isConvergent());
8500 std::pair<SDValue, SDValue>
Result =
8504 "Should've lowered as tail call");
8509 case Intrinsic::amdgcn_call_whole_wave: {
8511 bool isTailCall =
I.isTailCall();
8514 for (
unsigned Idx = 1;
Idx <
I.arg_size(); ++
Idx) {
8515 TargetLowering::ArgListEntry Arg(
getValue(
I.getArgOperand(Idx)),
8516 I.getArgOperand(Idx)->getType());
8517 Arg.setAttributes(&
I, Idx);
8524 Args.push_back(Arg);
8529 auto *Token = Bundle->Inputs[0].get();
8530 ConvControlToken =
getValue(Token);
8533 TargetLowering::CallLoweringInfo CLI(
DAG);
8537 getValue(
I.getArgOperand(0)), std::move(Args))
8541 .setConvergent(
I.isConvergent())
8542 .setConvergenceControlToken(ConvControlToken);
8545 std::pair<SDValue, SDValue>
Result =
8548 if (
Result.first.getNode())
8552 case Intrinsic::ptrmask: {
8568 auto HighOnes =
DAG.getNode(
8569 ISD::SHL, sdl, PtrVT,
DAG.getAllOnesConstant(sdl, PtrVT),
8570 DAG.getShiftAmountConstant(
Mask.getValueType().getFixedSizeInBits(),
8573 DAG.getZExtOrTrunc(Mask, sdl, PtrVT), HighOnes);
8574 }
else if (
Mask.getValueType() != PtrVT)
8575 Mask =
DAG.getPtrExtOrTrunc(Mask, sdl, PtrVT);
8581 case Intrinsic::threadlocal_address: {
8585 case Intrinsic::get_active_lane_mask: {
8589 EVT ElementVT =
Index.getValueType();
8600 SDValue VectorIndex =
DAG.getSplat(VecTy, sdl, Index);
8601 SDValue VectorTripCount =
DAG.getSplat(VecTy, sdl, TripCount);
8602 SDValue VectorStep =
DAG.getStepVector(sdl, VecTy);
8605 SDValue SetCC =
DAG.getSetCC(sdl, CCVT, VectorInduction,
8610 case Intrinsic::mask_beforefirst: {
8616 case Intrinsic::experimental_get_vector_length: {
8618 "Expected positive VF");
8623 EVT CountVT =
Count.getValueType();
8626 visitTargetIntrinsic(
I, Intrinsic);
8635 if (CountVT.
bitsLT(VT)) {
8640 SDValue MaxEVL =
DAG.getElementCount(sdl, CountVT,
8650 case Intrinsic::vector_partial_reduce_add: {
8655 Input,
DAG.getConstant(1, sdl,
Input.getValueType())));
8658 case Intrinsic::vector_partial_reduce_fadd: {
8666 DAG.getConstantFP(1.0, sdl,
Input.getValueType()),
8670 case Intrinsic::experimental_cttz_elts: {
8672 EVT OpVT =
Op.getValueType();
8679 SDValue AllZero =
DAG.getConstant(0, sdl, OpVT);
8688 case Intrinsic::vector_insert: {
8696 if (
Index.getValueType() != VectorIdxTy)
8697 Index =
DAG.getVectorIdxConstant(
Index->getAsZExtVal(), sdl);
8704 case Intrinsic::vector_extract: {
8712 if (
Index.getValueType() != VectorIdxTy)
8713 Index =
DAG.getVectorIdxConstant(
Index->getAsZExtVal(), sdl);
8719 case Intrinsic::experimental_vector_match: {
8723 EVT ResVT =
Mask.getValueType();
8727 case Intrinsic::vector_reverse:
8728 visitVectorReverse(
I);
8730 case Intrinsic::vector_splice_left:
8731 case Intrinsic::vector_splice_right:
8732 visitVectorSplice(
I);
8734 case Intrinsic::callbr_landingpad:
8735 visitCallBrLandingPad(
I);
8737 case Intrinsic::vector_interleave2:
8738 visitVectorInterleave(
I, 2);
8740 case Intrinsic::vector_interleave3:
8741 visitVectorInterleave(
I, 3);
8743 case Intrinsic::vector_interleave4:
8744 visitVectorInterleave(
I, 4);
8746 case Intrinsic::vector_interleave5:
8747 visitVectorInterleave(
I, 5);
8749 case Intrinsic::vector_interleave6:
8750 visitVectorInterleave(
I, 6);
8752 case Intrinsic::vector_interleave7:
8753 visitVectorInterleave(
I, 7);
8755 case Intrinsic::vector_interleave8:
8756 visitVectorInterleave(
I, 8);
8758 case Intrinsic::vector_deinterleave2:
8759 visitVectorDeinterleave(
I, 2);
8761 case Intrinsic::vector_deinterleave3:
8762 visitVectorDeinterleave(
I, 3);
8764 case Intrinsic::vector_deinterleave4:
8765 visitVectorDeinterleave(
I, 4);
8767 case Intrinsic::vector_deinterleave5:
8768 visitVectorDeinterleave(
I, 5);
8770 case Intrinsic::vector_deinterleave6:
8771 visitVectorDeinterleave(
I, 6);
8773 case Intrinsic::vector_deinterleave7:
8774 visitVectorDeinterleave(
I, 7);
8776 case Intrinsic::vector_deinterleave8:
8777 visitVectorDeinterleave(
I, 8);
8779 case Intrinsic::vector_repeat: {
8785 case Intrinsic::experimental_vector_compress:
8787 getValue(
I.getArgOperand(0)).getValueType(),
8792 case Intrinsic::vector_shuffle:
8798 case Intrinsic::experimental_convergence_anchor:
8799 case Intrinsic::experimental_convergence_entry:
8800 case Intrinsic::experimental_convergence_loop:
8801 visitConvergenceControl(
I, Intrinsic);
8803 case Intrinsic::experimental_vector_histogram_add: {
8804 visitVectorHistogram(
I, Intrinsic);
8807 case Intrinsic::experimental_vector_extract_last_active: {
8808 visitVectorExtractLastActive(
I, Intrinsic);
8811 case Intrinsic::loop_dependence_war_mask:
8816 DAG.getConstant(0, sdl, MVT::i64)));
8818 case Intrinsic::loop_dependence_raw_mask:
8823 DAG.getConstant(0, sdl, MVT::i64)));
8825 case Intrinsic::masked_udiv:
8831 case Intrinsic::masked_sdiv:
8837 case Intrinsic::masked_urem:
8843 case Intrinsic::masked_srem:
8852void SelectionDAGBuilder::pushFPOpOutChain(
SDValue Result,
8868 PendingConstrainedFP.push_back(OutChain);
8871 PendingConstrainedFPStrict.push_back(OutChain);
8876void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8890 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
8892 SDVTList VTs =
DAG.getVTList(VT, MVT::Other);
8896 Flags.setNoFPExcept(
true);
8904#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8905 case Intrinsic::INTRINSIC: \
8906 Opcode = ISD::STRICT_##DAGN; \
8908#include "llvm/IR/ConstrainedOps.def"
8909 case Intrinsic::experimental_constrained_fmuladd: {
8915 pushFPOpOutChain(
Mul, EB);
8938 if (
DAG.isKnownNeverNaN(Opers[1]) &&
DAG.isKnownNeverNaN(Opers[2]))
8946 pushFPOpOutChain(Result, EB);
8953 std::optional<unsigned> ResOPC;
8955 case Intrinsic::vp_cttz_elts: {
8957 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_POISON : ISD::VP_CTTZ_ELTS;
8960#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8961 case Intrinsic::VPID: \
8962 ResOPC = ISD::VPSD; \
8964#include "llvm/IR/VPIntrinsics.def"
8969 "Inconsistency: no SDNode available for this VPIntrinsic!");
8971 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8972 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8974 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8975 : ISD::VP_REDUCE_FMUL;
8981void SelectionDAGBuilder::visitVPLoad(
8996 SDValue InChain = AddToChain ?
DAG.getRoot() :
DAG.getEntryNode();
8997 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9000 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9001 MachinePointerInfo(PtrOperand), MMOFlags,
9003 MMOMetadata(AAInfo, Ranges));
9004 LD =
DAG.getLoadVP(VT,
DL, InChain, OpValues[0], OpValues[1], OpValues[2],
9011void SelectionDAGBuilder::visitVPLoadFF(
9014 assert(OpValues.
size() == 3 &&
"Unexpected number of operands");
9027 SDValue InChain = AddToChain ?
DAG.getRoot() :
DAG.getEntryNode();
9028 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9031 MMOMetadata(AAInfo, Ranges));
9032 LD =
DAG.getLoadFFVP(VT,
DL, InChain, OpValues[0], OpValues[1], OpValues[2],
9037 setValue(&VPIntrin,
DAG.getMergeValues({LD.getValue(0), Trunc},
DL));
9040void SelectionDAGBuilder::visitVPGather(
9044 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9056 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9058 *Alignment, MMOMetadata(AAInfo, Ranges));
9068 EVT IdxVT =
Index.getValueType();
9074 LD =
DAG.getGatherVP(
9075 DAG.getVTList(VT, MVT::Other), VT,
DL,
9076 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
9082void SelectionDAGBuilder::visitVPStore(
9086 EVT VT = OpValues[0].getValueType();
9094 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9097 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9098 MachinePointerInfo(PtrOperand), MMOFlags,
9107void SelectionDAGBuilder::visitVPScatter(
9110 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9112 EVT VT = OpValues[0].getValueType();
9122 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9124 *Alignment, AAInfo);
9134 EVT IdxVT =
Index.getValueType();
9140 ST =
DAG.getScatterVP(
DAG.getVTList(MVT::Other), VT,
DL,
9141 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
9142 OpValues[2], OpValues[3]},
9148void SelectionDAGBuilder::visitVPStridedLoad(
9160 SDValue InChain = AddToChain ?
DAG.getRoot() :
DAG.getEntryNode();
9162 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9165 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9167 *Alignment, MMOMetadata(AAInfo, Ranges));
9169 SDValue LD =
DAG.getStridedLoadVP(VT,
DL, InChain, OpValues[0], OpValues[1],
9170 OpValues[2], OpValues[3], MMO,
9178void SelectionDAGBuilder::visitVPStridedStore(
9182 EVT VT = OpValues[0].getValueType();
9188 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9191 MachineMemOperand *MMO =
DAG.getMachineFunction().getMachineMemOperand(
9193 *Alignment, AAInfo);
9197 DAG.getPOISON(OpValues[1].getValueType()), OpValues[2], OpValues[3],
9205void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
9213 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9215 SDVTList VTs =
DAG.getVTList(ValueVTs);
9221 "Unexpected target EVL type");
9225 for (
unsigned I = 0;
I < VPIntrin.
arg_size(); ++
I) {
9227 if (
I == EVLParamPos)
9234 SDNodeFlags SDFlags;
9242 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
9244 case ISD::VP_LOAD_FF:
9245 visitVPLoadFF(VPIntrin, ValueVTs[0], ValueVTs[1], OpValues);
9247 case ISD::VP_GATHER:
9248 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
9250 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
9251 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
9254 visitVPStore(VPIntrin, OpValues);
9256 case ISD::VP_SCATTER:
9257 visitVPScatter(VPIntrin, OpValues);
9259 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
9260 visitVPStridedStore(VPIntrin, OpValues);
9262 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
9263 case ISD::VP_CTTZ_ELTS: {
9265 DAG.getNode(Opcode,
DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
9281 BeginLabel =
nullptr;
9291 unsigned CallSiteIndex =
FuncInfo.getCurrentCallSite();
9292 if (CallSiteIndex) {
9322 assert(
II &&
"II should've been set");
9333std::pair<SDValue, SDValue>
9347 std::pair<SDValue, SDValue> Result =
TLI.LowerCallTo(CLI);
9350 "Non-null chain expected with non-tail call!");
9351 assert((Result.second.getNode() || !Result.first.getNode()) &&
9352 "Null value expected with tail call!");
9354 if (!Result.second.getNode()) {
9361 PendingExports.clear();
9363 DAG.setRoot(Result.second);
9381 if (!isMustTailCall &&
9382 Caller->getFnAttribute(
"disable-tail-calls").getValueAsBool())
9388 if (
DAG.hasSwiftErrorArg())
9397 bool isTailCall,
bool isMustTailCall,
9400 auto &
DL =
DAG.getDataLayout();
9407 const Value *SwiftErrorVal =
nullptr;
9414 const Value *V = *
I;
9417 if (V->getType()->isEmptyTy())
9422 Entry.setAttributes(&CB,
I - CB.
arg_begin());
9425 if (Entry.IsSwiftError &&
TLI.supportSwiftError()) {
9434 Args.push_back(Entry);
9445 Value *V = Bundle->Inputs[0];
9447 Entry.IsCFGuardTarget =
true;
9448 Args.push_back(Entry);
9453 if (
TLI.supportSwiftError() && SwiftErrorVal)
9459 if (!
TLI.supportKCFIBundles())
9461 "Target doesn't support calls with kcfi operand bundles.");
9469 auto *Token = Bundle->Inputs[0].get();
9470 ConvControlToken =
getValue(Token);
9481 .
setCallee(RetTy, FTy, Callee, std::move(Args), CB)
9492 if (!
TLI.supportPtrAuthBundles())
9494 "This target doesn't support calls with ptrauth operand bundles.");
9498 std::pair<SDValue, SDValue> Result =
lowerInvokable(CLI, EHPadBB);
9500 if (Result.first.getNode()) {
9509 if (SwiftErrorVal &&
TLI.supportSwiftError()) {
9515 DAG.setRoot(CopyNode);
9531 LoadTy, Builder.DAG.getDataLayout()))
9532 return Builder.getValue(LoadCst);
9538 bool ConstantMemory =
false;
9541 if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(PtrVal)) {
9542 Root = Builder.DAG.getEntryNode();
9543 ConstantMemory =
true;
9546 Root = Builder.DAG.getRoot();
9551 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
9554 if (!ConstantMemory)
9555 Builder.PendingLoads.push_back(LoadVal.
getValue(1));
9561void SelectionDAGBuilder::processIntegerCallValue(
const Instruction &
I,
9564 EVT VT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
9575bool SelectionDAGBuilder::visitMemCmpBCmpCall(
const CallInst &
I) {
9576 const Value *
LHS =
I.getArgOperand(0), *
RHS =
I.getArgOperand(1);
9577 const Value *
Size =
I.getArgOperand(2);
9580 EVT CallVT =
DAG.getTargetLoweringInfo().getValueType(
DAG.getDataLayout(),
9586 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9590 if (Res.first.getNode()) {
9591 processIntegerCallValue(
I, Res.first,
true);
9605 auto hasFastLoadsAndCompare = [&](
unsigned NumBits) {
9606 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
9628 switch (NumBitsToCompare) {
9640 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9653 LoadL =
DAG.getBitcast(CmpVT, LoadL);
9654 LoadR =
DAG.getBitcast(CmpVT, LoadR);
9658 processIntegerCallValue(
I, Cmp,
false);
9667bool SelectionDAGBuilder::visitMemChrCall(
const CallInst &
I) {
9668 const Value *Src =
I.getArgOperand(0);
9669 const Value *
Char =
I.getArgOperand(1);
9670 const Value *
Length =
I.getArgOperand(2);
9672 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9673 std::pair<SDValue, SDValue> Res =
9676 MachinePointerInfo(Src));
9677 if (Res.first.getNode()) {
9691bool SelectionDAGBuilder::visitMemCCpyCall(
const CallInst &
I) {
9692 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9699 processIntegerCallValue(
I, Res.first,
true);
9711bool SelectionDAGBuilder::visitMemPCpyCall(
const CallInst &
I) {
9716 Align DstAlign =
DAG.InferPtrAlign(Dst).valueOrOne();
9717 Align SrcAlign =
DAG.InferPtrAlign(Src).valueOrOne();
9726 Root, sdl, Dst, Src,
Size, DstAlign, SrcAlign,
false,
false,
9727 nullptr, std::nullopt, MachinePointerInfo(
I.getArgOperand(0)),
9728 MachinePointerInfo(
I.getArgOperand(1)),
I.getAAMetadata());
9730 "** memcpy should not be lowered as TailCall in mempcpy context **");
9734 Size =
DAG.getSExtOrTrunc(
Size, sdl, Dst.getValueType());
9747bool SelectionDAGBuilder::visitStrCpyCall(
const CallInst &
I,
bool isStpcpy) {
9748 const Value *Arg0 =
I.getArgOperand(0), *Arg1 =
I.getArgOperand(1);
9750 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9753 MachinePointerInfo(Arg0), MachinePointerInfo(Arg1), isStpcpy, &
I);
9754 if (Res.first.getNode()) {
9756 DAG.setRoot(Res.second);
9768bool SelectionDAGBuilder::visitStrCmpCall(
const CallInst &
I) {
9769 const Value *Arg0 =
I.getArgOperand(0), *Arg1 =
I.getArgOperand(1);
9771 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9774 MachinePointerInfo(Arg0), MachinePointerInfo(Arg1), &
I);
9775 if (Res.first.getNode()) {
9776 processIntegerCallValue(
I, Res.first,
true);
9789bool SelectionDAGBuilder::visitStrLenCall(
const CallInst &
I) {
9790 const Value *Arg0 =
I.getArgOperand(0);
9792 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9795 if (Res.first.getNode()) {
9796 processIntegerCallValue(
I, Res.first,
false);
9809bool SelectionDAGBuilder::visitStrNLenCall(
const CallInst &
I) {
9810 const Value *Arg0 =
I.getArgOperand(0), *Arg1 =
I.getArgOperand(1);
9812 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9813 std::pair<SDValue, SDValue> Res =
9816 MachinePointerInfo(Arg0));
9817 if (Res.first.getNode()) {
9818 processIntegerCallValue(
I, Res.first,
false);
9831bool SelectionDAGBuilder::visitStrstrCall(
const CallInst &
I) {
9832 const SelectionDAGTargetInfo &TSI =
DAG.getSelectionDAGInfo();
9833 const Value *Arg0 =
I.getArgOperand(0), *Arg1 =
I.getArgOperand(1);
9837 processIntegerCallValue(
I, Res.first,
false);
9849bool SelectionDAGBuilder::visitUnaryFloatCall(
const CallInst &
I,
9854 if (!
I.onlyReadsMemory() ||
I.isStrictFP())
9871bool SelectionDAGBuilder::visitBinaryFloatCall(
const CallInst &
I,
9876 if (!
I.onlyReadsMemory() ||
I.isStrictFP())
9889void SelectionDAGBuilder::visitCall(
const CallInst &
I) {
9891 if (
I.isInlineAsm()) {
9899 if (
F->isDeclaration()) {
9901 if (
unsigned IID =
F->getIntrinsicID()) {
9902 visitIntrinsicCall(
I, IID);
9912 LibFunc
Func = !
I.isNoBuiltin() && !
F->hasLocalLinkage() &&
F->hasName()
9915 if (
LibInfo->hasOptimizedCodeGen(Func)) {
9919 if (visitMemCmpBCmpCall(
I))
9959 case LibFunc_atan2f:
9960 case LibFunc_atan2l:
9985 case LibFunc_sqrt_finite:
9986 case LibFunc_sqrtf_finite:
9987 case LibFunc_sqrtl_finite:
10003 case LibFunc_exp10:
10004 case LibFunc_exp10f:
10005 case LibFunc_exp10l:
10009 case LibFunc_strstr:
10010 if (visitStrstrCall(
I))
10013 case LibFunc_memcmp:
10014 if (visitMemCmpBCmpCall(
I))
10017 case LibFunc_memccpy:
10018 if (visitMemCCpyCall(
I))
10021 case LibFunc_mempcpy:
10022 if (visitMemPCpyCall(
I))
10025 case LibFunc_memchr:
10026 if (visitMemChrCall(
I))
10029 case LibFunc_strcpy:
10030 if (visitStrCpyCall(
I,
false))
10033 case LibFunc_stpcpy:
10034 if (visitStrCpyCall(
I,
true))
10037 case LibFunc_strcmp:
10038 if (visitStrCmpCall(
I))
10041 case LibFunc_strlen:
10042 if (visitStrLenCall(
I))
10045 case LibFunc_strnlen:
10046 if (visitStrNLenCall(
I))
10070 if (
I.hasDeoptState())
10087 const Value *Discriminator = PAB->Inputs[1];
10089 assert(
Key->getType()->isIntegerTy(32) &&
"Invalid ptrauth key");
10090 assert(Discriminator->getType()->isIntegerTy(64) &&
10091 "Invalid ptrauth discriminator");
10096 if (CalleeCPA->isKnownCompatibleWith(
Key, Discriminator,
10097 DAG.getDataLayout()))
10137 for (
const auto &Code : Codes)
10152 SDISelAsmOperandInfo &MatchingOpInfo,
10154 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
10160 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
10161 TLI.getRegForInlineAsmConstraint(
TRI, OpInfo.ConstraintCode,
10162 OpInfo.ConstraintVT);
10163 std::pair<unsigned, const TargetRegisterClass *> InputRC =
10164 TLI.getRegForInlineAsmConstraint(
TRI, MatchingOpInfo.ConstraintCode,
10165 MatchingOpInfo.ConstraintVT);
10166 const bool OutOpIsIntOrFP =
10167 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
10168 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
10169 MatchingOpInfo.ConstraintVT.isFloatingPoint();
10170 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
10173 " with a matching output constraint of"
10174 " incompatible type!");
10176 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
10183 SDISelAsmOperandInfo &OpInfo,
10196 const Value *OpVal = OpInfo.CallOperandVal;
10214 DL.getPrefTypeAlign(Ty),
false,
10217 Chain = DAG.
getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
10219 TLI.getMemValueType(
DL, Ty));
10220 OpInfo.CallOperand = StackSlot;
10233static std::optional<unsigned>
10235 SDISelAsmOperandInfo &OpInfo,
10236 SDISelAsmOperandInfo &RefOpInfo) {
10247 return std::nullopt;
10251 unsigned AssignedReg;
10253 std::tie(AssignedReg, RC) =
TLI.getRegForInlineAsmConstraint(
10254 &
TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
10257 return std::nullopt;
10262 const MVT RegVT = *
TRI.legalclasstypes_begin(*RC);
10264 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
10273 !
TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
10278 if (RegVT.
getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
10283 OpInfo.CallOperand =
10285 OpInfo.ConstraintVT = RegVT;
10289 }
else if (RegVT.
isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
10293 return std::nullopt;
10295 OpInfo.CallOperand =
10297 OpInfo.ConstraintVT = VT;
10304 if (OpInfo.isMatchingInputConstraint())
10305 return std::nullopt;
10307 EVT ValueVT = OpInfo.ConstraintVT;
10308 if (OpInfo.ConstraintVT == MVT::Other)
10312 unsigned NumRegs = 1;
10313 if (OpInfo.ConstraintVT != MVT::Other)
10314 NumRegs =
TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
10328 I = std::find(
I, RC->
end(), AssignedReg);
10329 if (
I == RC->
end()) {
10332 return {AssignedReg};
10336 for (; NumRegs; --NumRegs, ++
I) {
10337 assert(
I != RC->
end() &&
"Ran out of registers to allocate!");
10342 OpInfo.AssignedRegs =
RegsForValue(Regs, RegVT, ValueVT);
10343 return std::nullopt;
10348 const std::vector<SDValue> &AsmNodeOperands) {
10351 for (; OperandNo; --OperandNo) {
10353 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
10356 (
F.isRegDefKind() ||
F.isRegDefEarlyClobberKind() ||
F.isMemKind()) &&
10357 "Skipped past definitions?");
10358 CurOp +=
F.getNumOperandRegisters() + 1;
10366 unsigned Flags = 0;
10369 explicit ExtraFlags(
const CallBase &
Call) {
10371 if (
IA->hasSideEffects())
10373 if (
IA->isAlignStack())
10375 if (
IA->canThrow())
10382 void update(
const TargetLowering::AsmOperandInfo &OpInfo) {
10398 unsigned get()
const {
return Flags; }
10422struct ConstraintDecisionInfo {
10424 std::vector<SDValue> AsmNodeOperands;
10426 bool HasSideEffect =
false;
10429 SmallVector<char> Buffer;
10430 raw_svector_ostream ErrorMsg;
10432 ConstraintDecisionInfo() : ErrorMsg(Buffer) {}
10442 ExtraFlags &ExtraInfo) {
10443 for (
auto &
T : TargetConstraints) {
10444 Info.ConstraintOperands.push_back(SDISelAsmOperandInfo(
T));
10445 SDISelAsmOperandInfo &OpInfo = Info.ConstraintOperands.back();
10447 if (OpInfo.CallOperandVal)
10448 OpInfo.CallOperand = Builder.getValue(OpInfo.CallOperandVal);
10450 if (!Info.HasSideEffect)
10451 Info.HasSideEffect = OpInfo.hasMemory(
TLI);
10463 Info.ErrorMsg <<
"constraint '" <<
T.ConstraintCode
10464 <<
"' expects an integer constant expression";
10468 ExtraInfo.update(
T);
10482 IA->collectAsmStrs(AsmStrs);
10485 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10493 if (OpInfo.hasMatchingInput()) {
10494 SDISelAsmOperandInfo &
Input =
10495 Info.ConstraintOperands[OpInfo.MatchingInput];
10500 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
10526 if (OpInfo.isIndirect &&
isFunction(OpInfo.CallOperand) &&
10527 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
10529 OpInfo.isIndirect =
false;
10536 !OpInfo.isIndirect) {
10537 assert((OpInfo.isMultipleAlternative ||
10539 "Can only indirectify direct input operands!");
10546 OpInfo.CallOperandVal =
nullptr;
10549 OpInfo.isIndirect =
true;
10561 SDLoc DL = Builder.getCurSDLoc();
10562 for (SDISelAsmOperandInfo &OpInfo : Info.ConstraintOperands) {
10564 SDISelAsmOperandInfo &RefOpInfo =
10565 OpInfo.isMatchingInputConstraint()
10566 ? Info.ConstraintOperands[OpInfo.getMatchedOperand()]
10572 const char *
RegName =
TRI.getName(*RegError);
10573 Info.ErrorMsg <<
"register '" <<
RegName <<
"' allocated for constraint '"
10574 << OpInfo.ConstraintCode
10575 <<
"' does not match required type";
10579 auto DetectWriteToReservedRegister = [&]() {
10584 if (
Reg.isPhysical() &&
TRI.isInlineAsmReadOnlyReg(MF,
Reg)) {
10585 Info.ErrorMsg <<
"write to reserved register '"
10586 <<
TRI.getRegAsmName(
Reg) <<
"'";
10595 !OpInfo.isMatchingInputConstraint())) &&
10596 "Only address as input operand is allowed.");
10598 switch (OpInfo.Type) {
10602 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10604 "Failed to convert memory constraint code to constraint id.");
10609 Info.AsmNodeOperands.push_back(
10611 Info.AsmNodeOperands.push_back(OpInfo.CallOperand);
10616 if (OpInfo.AssignedRegs.Regs.empty()) {
10617 Info.ErrorMsg <<
"could not allocate output register for "
10618 <<
"constraint '" << OpInfo.ConstraintCode <<
"'";
10622 if (DetectWriteToReservedRegister())
10627 OpInfo.AssignedRegs.AddInlineAsmOperands(
10630 false, 0,
DL, DAG, Info.AsmNodeOperands);
10636 SDValue InOperandVal = OpInfo.CallOperand;
10638 if (OpInfo.isMatchingInputConstraint()) {
10642 Info.AsmNodeOperands);
10644 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10645 if (OpInfo.isIndirect) {
10647 Info.ErrorMsg <<
"inline asm not supported yet: cannot handle "
10648 <<
"tied indirect register inputs";
10658 MVT RegVT = R->getSimpleValueType(0);
10661 RC =
MRI.getRegClass(TiedReg);
10662 }
else if (RegVT == MVT::Untyped) {
10663 RC =
TRI.getMinimalPhysRegClass(TiedReg);
10665 RC =
TLI.getRegClassFor(RegVT);
10668 TRI.getMinimalPhysRegClass(TiedReg);
10674 for (
unsigned I = 0,
E = Flag.getNumOperandRegisters();
I !=
E; ++
I)
10681 &Info.Glue, &
Call);
10683 OpInfo.getMatchedOperand(),
DL, DAG,
10684 Info.AsmNodeOperands);
10688 assert(Flag.isMemKind() &&
"Unknown matching constraint!");
10689 assert(Flag.getNumOperandRegisters() == 1 &&
10690 "Unexpected number of operands");
10694 Flag.clearMemConstraint();
10695 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10698 Info.AsmNodeOperands.push_back(Info.AsmNodeOperands[CurOp + 1]);
10709 std::vector<SDValue>
Ops;
10710 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
10715 Info.ErrorMsg <<
"value out of range for constraint '"
10716 << OpInfo.ConstraintCode <<
"'";
10720 Info.ErrorMsg <<
"invalid operand for inline asm constraint '"
10721 << OpInfo.ConstraintCode <<
"'";
10734 assert((OpInfo.isIndirect ||
10736 "Operand must be indirect to be a mem!");
10741 OpInfo.CallOperandVal
10743 ->getAddressSpace()
10745 "Memory operands expect pointer values");
10748 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10750 "Failed to convert memory constraint code to constraint id.");
10755 Info.AsmNodeOperands.push_back(
10757 Info.AsmNodeOperands.push_back(InOperandVal);
10763 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10765 "Failed to convert memory constraint code to constraint id.");
10769 SDValue AsmOp = InOperandVal;
10770 if (
TLI.supportsInlineAsmFuncOperands() &&
isFunction(InOperandVal)) {
10781 Info.AsmNodeOperands.push_back(
10783 Info.AsmNodeOperands.push_back(AsmOp);
10789 Info.ErrorMsg <<
"unknown asm constraint '" << OpInfo.ConstraintCode
10795 if (OpInfo.isIndirect) {
10796 Info.ErrorMsg <<
"cannot handle indirect register inputs yet for "
10797 <<
"constraint '" << OpInfo.ConstraintCode <<
"'";
10802 if (OpInfo.AssignedRegs.Regs.empty()) {
10803 Info.ErrorMsg <<
"could not allocate input reg for constraint '"
10804 << OpInfo.ConstraintCode <<
"'";
10808 if (DetectWriteToReservedRegister())
10811 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG,
DL, Info.Chain,
10812 &Info.Glue, &
Call);
10813 OpInfo.AssignedRegs.AddInlineAsmOperands(
10821 if (!OpInfo.AssignedRegs.Regs.empty())
10822 OpInfo.AssignedRegs.AddInlineAsmOperands(
10839 ExtraFlags ExtraInfo(
Call);
10842 Info.HasSideEffect = IA->hasSideEffects();
10848 Info.Chain = Info.HasSideEffect ? Builder.getRoot() : DAG.
getRoot();
10852 if (IsCallBr || EmitEHLabels)
10856 Info.Chain = Builder.getControlRoot();
10859 Info.Chain = Builder.lowerStartEH(Info.Chain, EHPadBB, Info.BeginLabel);
10865 Info.AsmNodeOperands.push_back(
SDValue());
10867 IA->getAsmString().data(),
TLI.getProgramPointerTy(DAG.
getDataLayout())));
10872 const MDNode *SrcLoc =
Call.getMetadata(
"srcloc");
10873 Info.AsmNodeOperands.push_back(DAG.
getMDNode(SrcLoc));
10877 Info.AsmNodeOperands.push_back(
10886void SelectionDAGBuilder::visitInlineAsm(
const CallBase &
Call,
10888 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
10890 DAG.getDataLayout(),
DAG.getSubtarget().getRegisterInfo(),
Call);
10893 "InvokeInst must have an EHPadBB");
10895 ConstraintDecisionInfo
Info;
10898 return emitInlineAsmError(
Call,
Info.ErrorMsg.str());
10906 Info.AsmNodeOperands.push_back(Glue);
10912 Info.AsmNodeOperands);
10924 ResultTypes = StructResult->elements();
10925 else if (!CallResultType->
isVoidTy())
10926 ResultTypes =
ArrayRef(CallResultType);
10928 auto CurResultType = ResultTypes.
begin();
10929 auto handleRegAssign = [&](
SDValue V) {
10930 assert(CurResultType != ResultTypes.
end() &&
"Unexpected value");
10931 assert((*CurResultType)->isSized() &&
"Unexpected unsized type");
10944 if (ResultVT !=
V.getValueType() &&
10947 else if (ResultVT !=
V.getValueType() && ResultVT.
isInteger() &&
10948 V.getValueType().isInteger()) {
10954 assert(ResultVT ==
V.getValueType() &&
"Asm result value mismatch!");
10960 for (SDISelAsmOperandInfo &OpInfo :
Info.ConstraintOperands) {
10964 if (OpInfo.AssignedRegs.
Regs.empty())
10967 switch (OpInfo.ConstraintType) {
10971 Chain, &Glue, &
Call);
10983 assert(
false &&
"Unexpected unknown constraint");
10987 if (OpInfo.isIndirect) {
10988 const Value *Ptr = OpInfo.CallOperandVal;
10989 assert(Ptr &&
"Expected value CallOperandVal for indirect asm operand");
10991 MachinePointerInfo(Ptr));
10998 handleRegAssign(
V);
11000 handleRegAssign(Val);
11006 if (!ResultValues.
empty()) {
11007 assert(CurResultType == ResultTypes.
end() &&
11008 "Mismatch in number of ResultTypes");
11010 "Mismatch in number of output operands in asm result");
11013 DAG.getVTList(ResultVTs), ResultValues);
11018 if (!OutChains.
empty())
11022 Chain = lowerEndEH(Chain,
II, EHPadBB,
Info.BeginLabel);
11025 if (ResultValues.
empty() ||
Info.HasSideEffect || !OutChains.
empty() ||
11027 DAG.setRoot(Chain);
11030void SelectionDAGBuilder::emitInlineAsmError(
const CallBase &
Call,
11031 const Twine &Message) {
11032 LLVMContext &Ctx = *
DAG.getContext();
11036 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
11040 if (ValueVTs.
empty())
11044 for (
const EVT &VT : ValueVTs)
11045 Ops.push_back(
DAG.getUNDEF(VT));
11050void SelectionDAGBuilder::visitVAStart(
const CallInst &
I) {
11054 DAG.getSrcValue(
I.getArgOperand(0))));
11057void SelectionDAGBuilder::visitVAArg(
const VAArgInst &
I) {
11058 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
11059 const DataLayout &
DL =
DAG.getDataLayout();
11063 DL.getABITypeAlign(
I.getType()).value());
11064 DAG.setRoot(
V.getValue(1));
11066 if (
I.getType()->isPointerTy())
11067 V =
DAG.getPtrExtOrTrunc(
11072void SelectionDAGBuilder::visitVAEnd(
const CallInst &
I) {
11076 DAG.getSrcValue(
I.getArgOperand(0))));
11079void SelectionDAGBuilder::visitVACopy(
const CallInst &
I) {
11084 DAG.getSrcValue(
I.getArgOperand(0)),
11085 DAG.getSrcValue(
I.getArgOperand(1))));
11091 std::optional<ConstantRange> CR =
getRange(
I);
11093 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
11096 APInt Hi = CR->getUnsignedMax();
11097 unsigned Bits = std::max(
Hi.getActiveBits(),
11105 DAG.getValueType(SmallVT));
11106 unsigned NumVals =
Op.getNode()->getNumValues();
11112 Ops.push_back(ZExt);
11113 for (
unsigned I = 1;
I != NumVals; ++
I)
11114 Ops.push_back(
Op.getValue(
I));
11116 return DAG.getMergeValues(
Ops,
SL);
11126 SDValue TestConst =
DAG.getTargetConstant(Classes,
SDLoc(), MVT::i32);
11134 for (
unsigned I = 0, E =
Ops.size();
I != E; ++
I) {
11137 MergeOp, TestConst);
11140 return DAG.getMergeValues(
Ops,
SL);
11151 unsigned ArgIdx,
unsigned NumArgs,
SDValue Callee,
Type *ReturnTy,
11154 Args.reserve(NumArgs);
11158 for (
unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
11159 ArgI != ArgE; ++ArgI) {
11160 const Value *V =
Call->getOperand(ArgI);
11162 assert(!V->getType()->isEmptyTy() &&
"Empty type passed to intrinsic.");
11165 Entry.setAttributes(
Call, ArgI);
11166 Args.push_back(Entry);
11171 .
setCallee(
Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
11200 for (
unsigned I = StartIdx;
I <
Call.arg_size();
I++) {
11209 Ops.push_back(Builder.getValue(
Call.getArgOperand(
I)));
11215void SelectionDAGBuilder::visitStackmap(
const CallInst &CI) {
11241 Ops.push_back(Chain);
11242 Ops.push_back(InGlue);
11249 assert(
ID.getValueType() == MVT::i64);
11251 DAG.getTargetConstant(
ID->getAsZExtVal(),
DL,
ID.getValueType());
11252 Ops.push_back(IDConst);
11258 Ops.push_back(ShadConst);
11264 SDVTList NodeTys =
DAG.getVTList(MVT::Other, MVT::Glue);
11268 Chain =
DAG.getCALLSEQ_END(Chain, 0, 0, InGlue,
DL);
11273 DAG.setRoot(Chain);
11276 FuncInfo.MF->getFrameInfo().setHasStackMap();
11280void SelectionDAGBuilder::visitPatchpoint(
const CallBase &CB,
11297 Callee =
DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
11300 Callee =
DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
11301 SDLoc(SymbolicCallee),
11302 SymbolicCallee->getValueType(0));
11312 "Not enough arguments provided to the patchpoint intrinsic");
11315 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
11319 TargetLowering::CallLoweringInfo CLI(
DAG);
11324 SDNode *CallEnd =
Result.second.getNode();
11333 "Expected a callseq node.");
11335 bool HasGlue =
Call->getGluedNode();
11360 Ops.push_back(Callee);
11366 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
11367 Ops.push_back(
DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
11370 Ops.push_back(
DAG.getTargetConstant((
unsigned)
CC, dl, MVT::i32));
11375 for (
unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i !=
e; ++i)
11386 if (IsAnyRegCC && HasDef) {
11388 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
11391 assert(ValueVTs.
size() == 1 &&
"Expected only one return value type.");
11396 NodeTys =
DAG.getVTList(ValueVTs);
11398 NodeTys =
DAG.getVTList(MVT::Other, MVT::Glue);
11415 if (IsAnyRegCC && HasDef) {
11418 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
11424 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
11427void SelectionDAGBuilder::visitVectorReduce(
const CallInst &
I,
11429 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
11432 if (
I.arg_size() > 1)
11437 SDNodeFlags SDFlags;
11441 switch (Intrinsic) {
11442 case Intrinsic::vector_reduce_fadd:
11450 case Intrinsic::vector_reduce_fmul:
11458 case Intrinsic::vector_reduce_add:
11461 case Intrinsic::vector_reduce_mul:
11464 case Intrinsic::vector_reduce_and:
11467 case Intrinsic::vector_reduce_or:
11470 case Intrinsic::vector_reduce_xor:
11473 case Intrinsic::vector_reduce_smax:
11476 case Intrinsic::vector_reduce_smin:
11479 case Intrinsic::vector_reduce_umax:
11482 case Intrinsic::vector_reduce_umin:
11485 case Intrinsic::vector_reduce_fmax:
11488 case Intrinsic::vector_reduce_fmin:
11491 case Intrinsic::vector_reduce_fmaximum:
11494 case Intrinsic::vector_reduce_fminimum:
11497 case Intrinsic::vector_reduce_fmaximumnum:
11500 case Intrinsic::vector_reduce_fminimumnum:
11514 Attrs.push_back(Attribute::SExt);
11516 Attrs.push_back(Attribute::ZExt);
11518 Attrs.push_back(Attribute::InReg);
11520 return AttributeList::get(CLI.
RetTy->
getContext(), AttributeList::ReturnIndex,
11528std::pair<SDValue, SDValue>
11542 "Only supported for non-aggregate returns");
11545 for (
Type *Ty : RetOrigTys)
11554 RetOrigTys.
swap(OldRetOrigTys);
11555 RetVTs.
swap(OldRetVTs);
11556 Offsets.swap(OldOffsets);
11558 for (
size_t i = 0, e = OldRetVTs.
size(); i != e; ++i) {
11559 EVT RetVT = OldRetVTs[i];
11560 uint64_t
Offset = OldOffsets[i];
11563 unsigned RegisterVTByteSZ = RegisterVT.
getSizeInBits() / 8;
11564 RetOrigTys.
append(NumRegs, OldRetOrigTys[i]);
11565 RetVTs.
append(NumRegs, RegisterVT);
11566 for (
unsigned j = 0; j != NumRegs; ++j)
11579 int DemoteStackIdx = -100;
11584 uint64_t TySize =
DL.getTypeAllocSize(CLI.
RetTy);
11592 ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);
11593 Entry.IsSRet =
true;
11594 Entry.Alignment = Alignment;
11606 for (
unsigned I = 0, E = RetVTs.
size();
I != E; ++
I) {
11608 if (NeedsRegBlock) {
11609 Flags.setInConsecutiveRegs();
11610 if (
I == RetVTs.
size() - 1)
11611 Flags.setInConsecutiveRegsLast();
11613 EVT VT = RetVTs[
I];
11617 for (
unsigned i = 0; i != NumRegs; ++i) {
11631 CLI.
Ins.push_back(Ret);
11640 if (Arg.IsSwiftError) {
11646 CLI.
Ins.push_back(Ret);
11654 for (
unsigned i = 0, e = Args.size(); i != e; ++i) {
11658 Type *FinalType = Args[i].Ty;
11659 if (Args[i].IsByVal)
11660 FinalType = Args[i].IndirectType;
11663 for (
unsigned Value = 0, NumValues = OrigArgTys.
size();
Value != NumValues;
11666 Type *ArgTy = OrigArgTy;
11667 if (Args[i].Ty != Args[i].OrigTy) {
11668 assert(
Value == 0 &&
"Only supported for non-aggregate arguments");
11669 ArgTy = Args[i].Ty;
11674 Args[i].Node.getResNo() +
Value);
11681 Flags.setOrigAlign(OriginalAlignment);
11686 Flags.setPointer();
11689 if (Args[i].IsZExt)
11691 if (Args[i].IsSExt)
11693 if (Args[i].IsNoExt)
11695 if (Args[i].IsInReg) {
11702 Flags.setHvaStart();
11708 if (Args[i].IsSRet)
11710 if (Args[i].IsSwiftSelf)
11711 Flags.setSwiftSelf();
11712 if (Args[i].IsSwiftAsync)
11713 Flags.setSwiftAsync();
11714 if (Args[i].IsSwiftError)
11715 Flags.setSwiftError();
11716 if (Args[i].IsCFGuardTarget)
11717 Flags.setCFGuardTarget();
11718 if (Args[i].IsByVal)
11720 if (Args[i].IsByRef)
11722 if (Args[i].IsPreallocated) {
11723 Flags.setPreallocated();
11731 if (Args[i].IsInAlloca) {
11732 Flags.setInAlloca();
11741 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11742 unsigned FrameSize =
DL.getTypeAllocSize(Args[i].IndirectType);
11743 Flags.setByValSize(FrameSize);
11746 if (
auto MA = Args[i].Alignment)
11750 }
else if (
auto MA = Args[i].Alignment) {
11753 MemAlign = OriginalAlignment;
11755 Flags.setMemAlign(MemAlign);
11756 if (Args[i].IsNest)
11759 Flags.setInConsecutiveRegs();
11767 if (Args[i].IsSExt)
11769 else if (Args[i].IsZExt)
11774 if (Args[i].IsReturned && !
Op.getValueType().isVector() &&
11779 Args[i].Ty->getPointerAddressSpace())) &&
11780 RetVTs.
size() == NumValues &&
"unexpected use of 'returned'");
11793 CLI.
RetZExt == Args[i].IsZExt))
11794 Flags.setReturned();
11800 for (
unsigned j = 0; j !=
NumParts; ++j) {
11806 j * Parts[j].
getValueType().getStoreSize().getKnownMinValue());
11815 CLI.
Outs.push_back(MyFlags);
11816 CLI.
OutVals.push_back(Parts[j]);
11819 if (NeedsRegBlock &&
Value == NumValues - 1)
11820 CLI.
Outs[CLI.
Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11832 "LowerCall didn't return a valid chain!");
11834 "LowerCall emitted a return value for a tail call!");
11836 "LowerCall didn't emit the correct number of values!");
11848 for (
unsigned i = 0, e = CLI.
Ins.size(); i != e; ++i) {
11849 assert(InVals[i].
getNode() &&
"LowerCall emitted a null value!");
11850 assert(
EVT(CLI.
Ins[i].VT) == InVals[i].getValueType() &&
11851 "LowerCall emitted a value with the wrong type!");
11861 unsigned NumValues = RetVTs.
size();
11862 ReturnValues.
resize(NumValues);
11869 for (
unsigned i = 0; i < NumValues; ++i) {
11876 DemoteStackIdx, Offsets[i]),
11878 ReturnValues[i] = L;
11879 Chains[i] = L.getValue(1);
11886 std::optional<ISD::NodeType> AssertOp;
11891 unsigned CurReg = 0;
11892 for (
EVT VT : RetVTs) {
11898 CLI.
DAG, CLI.
DL, &InVals[CurReg], NumRegs, RegisterVT, VT,
nullptr,
11906 if (ReturnValues.
empty())
11912 return std::make_pair(Res, CLI.
Chain);
11929 if (
N->getNumValues() == 1) {
11937 "Lowering returned the wrong number of results!");
11940 for (
unsigned I = 0, E =
N->getNumValues();
I != E; ++
I)
11954 "Copy from a reg to the same reg!");
11955 assert(!Reg.isPhysical() &&
"Is a physreg");
11966 auto PreferredExtendIt =
FuncInfo.PreferredExtendType.find(V);
11967 if (PreferredExtendIt !=
FuncInfo.PreferredExtendType.end())
11968 ExtendType = PreferredExtendIt->second;
11971 PendingExports.push_back(Chain);
11983 return A->use_empty();
11985 const BasicBlock &Entry =
A->getParent()->front();
11986 for (
const User *U :
A->users())
11995 std::pair<const AllocaInst *, const StoreInst *>>;
12007 enum StaticAllocaInfo {
Unknown, Clobbered, Elidable };
12009 unsigned NumArgs = FuncInfo->
Fn->
arg_size();
12010 StaticAllocas.
reserve(NumArgs * 2);
12012 auto GetInfoIfStaticAlloca = [&](
const Value *
V) -> StaticAllocaInfo * {
12015 V =
V->stripPointerCasts();
12017 if (!AI || !AI->isStaticAlloca() || !FuncInfo->
StaticAllocaMap.count(AI))
12020 return &Iter.first->second;
12040 for (
const Use &U :
I.operands()) {
12041 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
12042 *Info = StaticAllocaInfo::Clobbered;
12048 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(
SI->getValueOperand()))
12049 *Info = StaticAllocaInfo::Clobbered;
12052 const Value *Dst =
SI->getPointerOperand()->stripPointerCasts();
12053 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
12059 if (*Info != StaticAllocaInfo::Unknown)
12067 const Value *Val =
SI->getValueOperand()->stripPointerCasts();
12070 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
12072 DL.getTypeStoreSize(Arg->
getType()) != *AllocaSize ||
12073 !
DL.typeSizeEqualsStoreSize(Arg->
getType()) ||
12074 ArgCopyElisionCandidates.count(Arg)) {
12075 *Info = StaticAllocaInfo::Clobbered;
12079 LLVM_DEBUG(
dbgs() <<
"Found argument copy elision candidate: " << *AI
12083 *Info = StaticAllocaInfo::Elidable;
12084 ArgCopyElisionCandidates.insert({Arg, {AI,
SI}});
12089 if (ArgCopyElisionCandidates.size() == NumArgs)
12113 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
12114 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
12115 const AllocaInst *AI = ArgCopyIter->second.first;
12116 int FixedIndex = FINode->getIndex();
12118 int OldIndex = AllocaIndex;
12122 dbgs() <<
" argument copy elision failed due to bad fixed stack "
12128 LLVM_DEBUG(
dbgs() <<
" argument copy elision failed: alignment of alloca "
12129 "greater than stack argument alignment ("
12130 <<
DebugStr(RequiredAlignment) <<
" vs "
12138 dbgs() <<
"Eliding argument copy from " << Arg <<
" to " << *AI <<
'\n'
12139 <<
" Replacing frame index " << OldIndex <<
" with " << FixedIndex
12145 AllocaIndex = FixedIndex;
12146 ArgCopyElisionFrameIndexMap.
insert({OldIndex, FixedIndex});
12147 for (
SDValue ArgVal : ArgVals)
12151 const StoreInst *
SI = ArgCopyIter->second.second;
12165 SelectionDAG &DAG =
SDB->DAG;
12166 SDLoc dl =
SDB->getCurSDLoc();
12171 if (
F.hasFnAttribute(Attribute::Naked))
12176 MVT ValueVT =
TLI->getPointerTy(
DL,
DL.getAllocaAddrSpace());
12178 ISD::ArgFlagsTy
Flags;
12180 MVT RegisterVT =
TLI->getRegisterType(*DAG.
getContext(), ValueVT);
12181 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT,
F.getReturnType(),
true,
12191 ArgCopyElisionCandidates);
12194 for (
const Argument &Arg :
F.args()) {
12195 unsigned ArgNo = Arg.getArgNo();
12198 bool isArgValueUsed = !Arg.
use_empty();
12200 if (Arg.hasAttribute(Attribute::ByVal))
12201 FinalType = Arg.getParamByValType();
12202 bool NeedsRegBlock =
TLI->functionArgumentNeedsConsecutiveRegisters(
12203 FinalType,
F.getCallingConv(),
F.isVarArg(),
DL);
12204 for (
unsigned Value = 0, NumValues =
Types.size();
Value != NumValues;
12207 EVT VT =
TLI->getValueType(
DL, ArgTy);
12208 ISD::ArgFlagsTy
Flags;
12211 Flags.setPointer();
12214 if (Arg.hasAttribute(Attribute::ZExt))
12216 if (Arg.hasAttribute(Attribute::SExt))
12218 if (Arg.hasAttribute(Attribute::InReg)) {
12225 Flags.setHvaStart();
12231 if (Arg.hasAttribute(Attribute::StructRet))
12233 if (Arg.hasAttribute(Attribute::SwiftSelf))
12234 Flags.setSwiftSelf();
12235 if (Arg.hasAttribute(Attribute::SwiftAsync))
12236 Flags.setSwiftAsync();
12237 if (Arg.hasAttribute(Attribute::SwiftError))
12238 Flags.setSwiftError();
12239 if (Arg.hasAttribute(Attribute::ByVal))
12241 if (Arg.hasAttribute(Attribute::ByRef))
12243 if (Arg.hasAttribute(Attribute::InAlloca)) {
12244 Flags.setInAlloca();
12252 if (Arg.hasAttribute(Attribute::Preallocated)) {
12253 Flags.setPreallocated();
12265 const Align OriginalAlignment(
12266 TLI->getABIAlignmentForCallingConv(ArgTy,
DL));
12267 Flags.setOrigAlign(OriginalAlignment);
12270 Type *ArgMemTy =
nullptr;
12271 if (
Flags.isByVal() ||
Flags.isInAlloca() ||
Flags.isPreallocated() ||
12274 ArgMemTy = Arg.getPointeeInMemoryValueType();
12276 uint64_t MemSize =
DL.getTypeAllocSize(ArgMemTy);
12281 if (
auto ParamAlign = Arg.getParamStackAlign())
12282 MemAlign = *ParamAlign;
12283 else if ((ParamAlign = Arg.getParamAlign()))
12284 MemAlign = *ParamAlign;
12286 MemAlign =
TLI->getByValTypeAlignment(ArgMemTy,
DL);
12287 if (
Flags.isByRef())
12288 Flags.setByRefSize(MemSize);
12290 Flags.setByValSize(MemSize);
12291 }
else if (
auto ParamAlign = Arg.getParamStackAlign()) {
12292 MemAlign = *ParamAlign;
12294 MemAlign = OriginalAlignment;
12296 Flags.setMemAlign(MemAlign);
12298 if (Arg.hasAttribute(Attribute::Nest))
12301 Flags.setInConsecutiveRegs();
12302 if (ArgCopyElisionCandidates.count(&Arg))
12303 Flags.setCopyElisionCandidate();
12304 if (Arg.hasAttribute(Attribute::Returned))
12305 Flags.setReturned();
12307 MVT RegisterVT =
TLI->getRegisterTypeForCallingConv(
12308 *
CurDAG->getContext(),
F.getCallingConv(), VT);
12309 unsigned NumRegs =
TLI->getNumRegistersForCallingConv(
12310 *
CurDAG->getContext(),
F.getCallingConv(), VT);
12311 for (
unsigned i = 0; i != NumRegs; ++i) {
12315 ISD::InputArg MyFlags(
12316 Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,
12318 if (NumRegs > 1 && i == 0)
12319 MyFlags.Flags.setSplit();
12322 MyFlags.Flags.setOrigAlign(
Align(1));
12323 if (i == NumRegs - 1)
12324 MyFlags.Flags.setSplitEnd();
12328 if (NeedsRegBlock &&
Value == NumValues - 1)
12329 Ins[Ins.
size() - 1].Flags.setInConsecutiveRegsLast();
12335 SDValue NewRoot =
TLI->LowerFormalArguments(
12336 DAG.
getRoot(),
F.getCallingConv(),
F.isVarArg(), Ins, dl, DAG, InVals);
12340 "LowerFormalArguments didn't return a valid chain!");
12342 "LowerFormalArguments didn't emit the correct number of values!");
12344 "LowerFormalArguments emitted a null value!");
12354 MVT VT =
TLI->getPointerTy(
DL,
DL.getAllocaAddrSpace());
12355 MVT RegVT =
TLI->getRegisterType(*
CurDAG->getContext(), VT);
12356 std::optional<ISD::NodeType> AssertOp;
12359 F.getCallingConv(), AssertOp);
12362 MachineRegisterInfo&
RegInfo =
MF.getRegInfo();
12364 RegInfo.createVirtualRegister(
TLI->getRegClassFor(RegVT));
12365 FuncInfo->DemoteRegister = SRetReg;
12367 SDB->DAG.getCopyToReg(NewRoot,
SDB->getCurSDLoc(), SRetReg, ArgValue);
12375 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
12376 for (
const Argument &Arg :
F.args()) {
12380 unsigned NumValues = ValueVTs.
size();
12381 if (NumValues == 0)
12388 if (Ins[i].
Flags.isCopyElisionCandidate()) {
12390 for (EVT VT : ValueVTs)
12392 F.getCallingConv(), VT);
12401 bool isSwiftErrorArg =
12402 TLI->supportSwiftError() &&
12403 Arg.hasAttribute(Attribute::SwiftError);
12404 if (!ArgHasUses && !isSwiftErrorArg) {
12405 SDB->setUnusedArgValue(&Arg, InVals[i]);
12408 if (FrameIndexSDNode *FI =
12410 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12413 for (
unsigned Val = 0; Val != NumValues; ++Val) {
12414 EVT VT = ValueVTs[Val];
12415 MVT PartVT =
TLI->getRegisterTypeForCallingConv(*
CurDAG->getContext(),
12416 F.getCallingConv(), VT);
12417 unsigned NumParts =
TLI->getNumRegistersForCallingConv(
12418 *
CurDAG->getContext(),
F.getCallingConv(), VT);
12423 if (ArgHasUses || isSwiftErrorArg) {
12424 std::optional<ISD::NodeType> AssertOp;
12425 if (Arg.hasAttribute(Attribute::SExt))
12427 else if (Arg.hasAttribute(Attribute::ZExt))
12432 NewRoot,
F.getCallingConv(), AssertOp);
12435 if (NoFPClass !=
fcNone) {
12437 static_cast<uint64_t>(NoFPClass), dl, MVT::i32);
12439 OutVal, SDNoFPClass);
12448 if (ArgValues.
empty())
12452 if (FrameIndexSDNode *FI =
12454 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12457 SDB->getCurSDLoc());
12459 SDB->setValue(&Arg, Res);
12469 if (LoadSDNode *LNode =
12471 if (FrameIndexSDNode *FI =
12473 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
12501 FuncInfo->InitializeRegForValue(&Arg);
12502 SDB->CopyToExportRegsIfNeeded(&Arg);
12506 if (!Chains.
empty()) {
12513 assert(i == InVals.
size() &&
"Argument register count mismatch!");
12517 if (!ArgCopyElisionFrameIndexMap.
empty()) {
12518 for (MachineFunction::VariableDbgInfo &VI :
12519 MF->getInStackSlotVariableDbgInfo()) {
12520 auto I = ArgCopyElisionFrameIndexMap.
find(
VI.getStackSlot());
12521 if (
I != ArgCopyElisionFrameIndexMap.
end())
12522 VI.updateStackSlot(
I->second);
12537SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(
const BasicBlock *LLVMBB) {
12538 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
12540 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
12546 MachineBasicBlock *SuccMBB =
FuncInfo.getMBB(SuccBB);
12550 if (!SuccsHandled.
insert(SuccMBB).second)
12558 for (
const PHINode &PN : SuccBB->phis()) {
12560 if (PN.use_empty())
12564 if (PN.getType()->isEmptyTy())
12568 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
12573 RegOut =
FuncInfo.CreateRegs(PHIOp);
12584 auto I =
FuncInfo.ValueMap.find(PHIOp);
12590 "Didn't codegen value into a register!??");
12600 for (EVT VT : ValueVTs) {
12602 for (
unsigned i = 0; i != NumRegisters; ++i)
12604 Reg += NumRegisters;
12624void SelectionDAGBuilder::updateDAGForMaybeTailCall(
SDValue MaybeTC) {
12626 if (MaybeTC.
getNode() !=
nullptr)
12627 DAG.setRoot(MaybeTC);
12632void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W,
Value *
Cond,
12636 MachineBasicBlock *NextMBB =
nullptr;
12641 unsigned Size =
W.LastCluster -
W.FirstCluster + 1;
12643 BranchProbabilityInfo *BPI =
FuncInfo.BPI;
12645 if (
Size == 2 &&
W.MBB == SwitchMBB) {
12653 CaseCluster &
Small = *
W.FirstCluster;
12654 CaseCluster &
Big = *
W.LastCluster;
12658 const APInt &SmallValue =
Small.Low->getValue();
12659 const APInt &BigValue =
Big.Low->getValue();
12662 APInt CommonBit = BigValue ^ SmallValue;
12669 DAG.getConstant(CommonBit,
DL, VT));
12671 DL, MVT::i1,
Or,
DAG.getConstant(BigValue | SmallValue,
DL, VT),
12677 addSuccessorWithProb(SwitchMBB,
Small.MBB,
Small.Prob +
Big.Prob);
12679 addSuccessorWithProb(
12680 SwitchMBB, DefaultMBB,
12684 addSuccessorWithProb(SwitchMBB, DefaultMBB);
12692 DAG.getBasicBlock(DefaultMBB));
12694 DAG.setRoot(BrCond);
12706 [](
const CaseCluster &a,
const CaseCluster &b) {
12707 return a.Prob != b.Prob ?
12709 a.Low->getValue().slt(b.Low->getValue());
12716 if (
I->Prob >
W.LastCluster->Prob)
12718 if (
I->Kind ==
CC_Range &&
I->MBB == NextMBB) {
12726 BranchProbability DefaultProb =
W.DefaultProb;
12727 BranchProbability UnhandledProbs = DefaultProb;
12729 UnhandledProbs +=
I->Prob;
12731 MachineBasicBlock *CurMBB =
W.MBB;
12733 bool FallthroughUnreachable =
false;
12734 MachineBasicBlock *Fallthrough;
12735 if (
I ==
W.LastCluster) {
12737 Fallthrough = DefaultMBB;
12738 FallthroughUnreachable =
12742 CurMF->
insert(BBI, Fallthrough);
12746 UnhandledProbs -=
I->Prob;
12751 JumpTableHeader *JTH = &
SL->JTCases[
I->JTCasesIndex].first;
12752 SwitchCG::JumpTable *JT = &
SL->JTCases[
I->JTCasesIndex].second;
12755 MachineBasicBlock *JumpMBB = JT->
MBB;
12756 CurMF->
insert(BBI, JumpMBB);
12758 auto JumpProb =
I->Prob;
12759 auto FallthroughProb = UnhandledProbs;
12767 if (*SI == DefaultMBB) {
12768 JumpProb += DefaultProb / 2;
12769 FallthroughProb -= DefaultProb / 2;
12788 if (FallthroughUnreachable) {
12790 bool HasCFBranch =
false;
12793 HasCFBranch = !CFBranch->isZero();
12794 if (!HasCFBranch &&
12800 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
12801 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
12810 if (CurMBB == SwitchMBB) {
12818 BitTestBlock *BTB = &
SL->BitTestCases[
I->BTCasesIndex];
12821 for (BitTestCase &BTC : BTB->
Cases)
12833 BTB->
Prob += DefaultProb / 2;
12837 if (FallthroughUnreachable)
12841 if (CurMBB == SwitchMBB) {
12848 const Value *
RHS, *
LHS, *MHS;
12850 if (
I->Low ==
I->High) {
12865 if (FallthroughUnreachable)
12869 CaseBlock CB(
CC,
LHS,
RHS, MHS,
I->MBB, Fallthrough, CurMBB,
12872 if (CurMBB == SwitchMBB)
12875 SL->SwitchCases.push_back(CB);
12880 CurMBB = Fallthrough;
12884void SelectionDAGBuilder::splitWorkItem(
SwitchWorkList &WorkList,
12885 const SwitchWorkListItem &W,
12888 assert(
W.FirstCluster->Low->getValue().slt(
W.LastCluster->Low->getValue()) &&
12889 "Clusters not sorted?");
12890 assert(
W.LastCluster -
W.FirstCluster + 1 >= 2 &&
"Too small to split!");
12892 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12893 SL->computeSplitWorkItemInfo(W);
12898 assert(PivotCluster >
W.FirstCluster);
12899 assert(PivotCluster <=
W.LastCluster);
12904 const ConstantInt *Pivot = PivotCluster->Low;
12913 MachineBasicBlock *LeftMBB;
12914 if (FirstLeft == LastLeft && FirstLeft->Kind ==
CC_Range &&
12915 FirstLeft->Low ==
W.GE &&
12916 (FirstLeft->High->getValue() + 1LL) == Pivot->
getValue()) {
12917 LeftMBB = FirstLeft->MBB;
12919 LeftMBB =
FuncInfo.MF->CreateMachineBasicBlock(
W.MBB->getBasicBlock());
12920 FuncInfo.MF->insert(BBI, LeftMBB);
12922 {LeftMBB, FirstLeft, LastLeft,
W.GE, Pivot,
W.DefaultProb / 2});
12930 MachineBasicBlock *RightMBB;
12931 if (FirstRight == LastRight && FirstRight->Kind ==
CC_Range &&
12932 W.LT && (FirstRight->High->getValue() + 1ULL) ==
W.LT->getValue()) {
12933 RightMBB = FirstRight->MBB;
12935 RightMBB =
FuncInfo.MF->CreateMachineBasicBlock(
W.MBB->getBasicBlock());
12936 FuncInfo.MF->insert(BBI, RightMBB);
12938 {RightMBB, FirstRight, LastRight, Pivot,
W.LT,
W.DefaultProb / 2});
12944 CaseBlock CB(
ISD::SETLT,
Cond, Pivot,
nullptr, LeftMBB, RightMBB,
W.MBB,
12947 if (
W.MBB == SwitchMBB)
12950 SL->SwitchCases.push_back(CB);
12975 MachineBasicBlock *SwitchMBB =
FuncInfo.MBB;
12983 unsigned PeeledCaseIndex = 0;
12984 bool SwitchPeeled =
false;
12985 for (
unsigned Index = 0;
Index < Clusters.size(); ++
Index) {
12986 CaseCluster &
CC = Clusters[
Index];
12987 if (
CC.Prob < TopCaseProb)
12989 TopCaseProb =
CC.Prob;
12990 PeeledCaseIndex =
Index;
12991 SwitchPeeled =
true;
12996 LLVM_DEBUG(
dbgs() <<
"Peeled one top case in switch stmt, prob: "
12997 << TopCaseProb <<
"\n");
13002 MachineBasicBlock *PeeledSwitchMBB =
13004 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
13007 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
13008 SwitchWorkListItem
W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
13009 nullptr,
nullptr, TopCaseProb.
getCompl()};
13010 lowerWorkItem(W,
SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
13012 Clusters.erase(PeeledCaseIt);
13013 for (CaseCluster &
CC : Clusters) {
13015 dbgs() <<
"Scale the probablity for one cluster, before scaling: "
13016 <<
CC.Prob <<
"\n");
13020 PeeledCaseProb = TopCaseProb;
13021 return PeeledSwitchMBB;
13024void SelectionDAGBuilder::visitSwitch(
const SwitchInst &
SI) {
13026 BranchProbabilityInfo *BPI =
FuncInfo.BPI;
13028 Clusters.reserve(
SI.getNumCases());
13029 for (
auto I :
SI.cases()) {
13030 MachineBasicBlock *Succ =
FuncInfo.getMBB(
I.getCaseSuccessor());
13031 const ConstantInt *CaseVal =
I.getCaseValue();
13032 BranchProbability Prob =
13034 : BranchProbability(1,
SI.getNumCases() + 1);
13038 MachineBasicBlock *DefaultMBB =
FuncInfo.getMBB(
SI.getDefaultDest());
13047 MachineBasicBlock *PeeledSwitchMBB =
13048 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
13051 MachineBasicBlock *SwitchMBB =
FuncInfo.MBB;
13052 if (Clusters.empty()) {
13053 assert(PeeledSwitchMBB == SwitchMBB);
13055 if (DefaultMBB != NextBlock(SwitchMBB)) {
13062 SL->findJumpTables(Clusters, &SI,
getCurSDLoc(), DefaultMBB,
DAG.getPSI(),
13064 SL->findBitTestClusters(Clusters, &SI);
13067 dbgs() <<
"Case clusters: ";
13068 for (
const CaseCluster &
C : Clusters) {
13074 C.Low->getValue().print(
dbgs(),
true);
13075 if (
C.Low !=
C.High) {
13077 C.High->getValue().print(
dbgs(),
true);
13084 assert(!Clusters.empty());
13088 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
13092 DefaultMBB ==
FuncInfo.getMBB(
SI.getDefaultDest()))
13095 {PeeledSwitchMBB,
First,
Last,
nullptr,
nullptr, DefaultProb});
13097 while (!WorkList.
empty()) {
13099 unsigned NumClusters =
W.LastCluster -
W.FirstCluster + 1;
13104 splitWorkItem(WorkList, W,
SI.getCondition(), SwitchMBB);
13108 lowerWorkItem(W,
SI.getCondition(), SwitchMBB, DefaultMBB);
13112void SelectionDAGBuilder::visitStepVector(
const CallInst &
I) {
13113 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
13119void SelectionDAGBuilder::visitVectorReverse(
const CallInst &
I) {
13120 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
13125 assert(VT ==
V.getValueType() &&
"Malformed vector.reverse!");
13134 SmallVector<int, 8>
Mask;
13136 for (
unsigned i = 0; i != NumElts; ++i)
13137 Mask.push_back(NumElts - 1 - i);
13142void SelectionDAGBuilder::visitVectorDeinterleave(
const CallInst &
I,
13151 EVT OutVT = ValueVTs[0];
13155 for (
unsigned i = 0; i != Factor; ++i) {
13156 assert(ValueVTs[i] == OutVT &&
"Expected VTs to be the same");
13158 DAG.getVectorIdxConstant(OutNumElts * i,
DL));
13162 DAG.getVTList(ValueVTs), SubVecs);
13166void SelectionDAGBuilder::visitVectorInterleave(
const CallInst &
I,
13169 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
13174 for (
unsigned i = 0; i < Factor; ++i) {
13177 "Expected VTs to be the same");
13185 for (
unsigned i = 0; i < Factor; ++i)
13192void SelectionDAGBuilder::visitFreeze(
const FreezeInst &
I) {
13196 unsigned NumValues = ValueVTs.
size();
13197 if (NumValues == 0)
return;
13202 for (
unsigned i = 0; i != NumValues; ++i)
13210void SelectionDAGBuilder::visitVectorSplice(
const CallInst &
I) {
13211 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
13217 const bool IsLeft =
I.getIntrinsicID() == Intrinsic::vector_splice_left;
13235 SmallVector<int, 8>
Mask;
13236 for (
unsigned i = 0; i < NumElts; ++i)
13237 Mask.push_back(Idx + i);
13265 assert(
MI->getOpcode() == TargetOpcode::COPY &&
13266 "start of copy chain MUST be COPY");
13267 Reg =
MI->getOperand(1).getReg();
13270 assert(
Reg.isVirtual() &&
"expected COPY of virtual register");
13271 MI =
MRI.def_begin(
Reg)->getParent();
13274 if (
MI->getOpcode() == TargetOpcode::COPY) {
13275 assert(
Reg.isVirtual() &&
"expected COPY of virtual register");
13276 Reg =
MI->getOperand(1).getReg();
13277 assert(
Reg.isPhysical() &&
"expected COPY of physical register");
13280 assert(
MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
13281 "end of copy chain MUST be INLINEASM_BR");
13291void SelectionDAGBuilder::visitCallBrLandingPad(
const CallInst &
I) {
13297 const TargetLowering &
TLI =
DAG.getTargetLoweringInfo();
13298 const TargetRegisterInfo *
TRI =
DAG.getSubtarget().getRegisterInfo();
13299 MachineRegisterInfo &
MRI =
DAG.getMachineFunction().getRegInfo();
13307 for (
auto &
T : TargetConstraints) {
13308 SDISelAsmOperandInfo OpInfo(
T);
13316 switch (OpInfo.ConstraintType) {
13327 FuncInfo.MBB->addLiveIn(OriginalDef);
13335 ResultVTs.
push_back(OpInfo.ConstraintVT);
13344 ResultVTs.
push_back(OpInfo.ConstraintVT);
13352 DAG.getVTList(ResultVTs), ResultValues);
static unsigned getIntrinsicID(const SDNode *N)
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This file implements the BitVector class.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
static Value * getCondition(Instruction *I)
const HexagonInstrInfo * TII
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
static void computeConstraintToUse(const TargetLowering *TLI, TargetLowering::AsmOperandInfo &OpInfo)
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isUndef(const MachineInstr &MI)
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static const Function * getCalledFunction(const Value *V)
This file provides utility analysis objects describing memory locations.
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
OptimizedStructLayoutField Field
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static void failForInvalidBundles(const CallBase &I, StringRef Name, ArrayRef< uint32_t > AllowedBundles)
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info, const CallBase &Call, SelectionDAGBuilder &Builder, const TargetLowering &TLI, SelectionDAG &DAG)
Prepare DAG-level operands.
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static const GlobalValue * getGlobalAddressDbgOperand(const Value *V, DIExpression *&Expr, unsigned OpIdx, const MachineFunction &MF)
If V is the address of a describable global, possibly displaced by a constant, return the global and ...
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
DenseMap< const Argument *, std::pair< const AllocaInst *, const StoreInst * > > ArgCopyElisionMapTy
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< Register, TypeSize > > &Regs, const SDValue &N)
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static bool determineConstraints(ConstraintDecisionInfo &Info, TargetLowering::AsmOperandInfoVector &TargetConstraints, const CallBase &Call, SelectionDAGBuilder &Builder, const TargetLowering &TLI, const TargetMachine &TM, SelectionDAG &DAG, const BasicBlock *EHPadBB)
DetermineConstraints - Find the constraints to use for inline asm operands.
static bool constructOperandInfo(ConstraintDecisionInfo &Info, TargetLowering::AsmOperandInfoVector &TargetConstraints, SelectionDAGBuilder &Builder, const TargetLowering &TLI, ExtraFlags &ExtraInfo)
Construct operand info objects.
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static void ensureMatchedVecRegParts(unsigned NumRegs, unsigned NumParts, MVT RegisterVT, MVT PartVT)
Emit a fatal error if the broken down registers don't match part type or count.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
static FPClassTest getNoFPClass(const Instruction &I)
static LLVM_ATTRIBUTE_ALWAYS_INLINE MVT::SimpleValueType getSimpleVT(const uint8_t *MatcherTable, size_t &MatcherIndex)
getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value, use GetVBR to decode it.
This file defines the SmallPtrSet class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
uint16_t RegSizeInBits(const MCRegisterInfo &MRI, MCRegister RegNo)
static const fltSemantics & IEEEsingle()
static LLVM_ABI Semantics SemanticsToEnum(const llvm::fltSemantics &Sem)
static LLVM_ABI const fltSemantics * getArbitraryFPSemantics(StringRef Format)
Returns the fltSemantics for a given arbitrary FP format string, or nullptr if invalid.
Class for arbitrary precision integers.
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
an instruction to allocate memory on the stack
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
LLVM_ABI std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
This class represents an incoming formal argument to a Function.
LLVM_ABI bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
This class holds the attributes for a particular argument, parameter, function, or return value.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
InstListType::const_iterator const_iterator
LLVM_ABI bool isEntryBlock() const
Return true if this is the entry block of the containing function.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
This class represents a no-op cast from one type to another.
The address of a basic block.
Analysis providing branch probability information.
LLVM_ABI BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
LLVM_ABI bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static constexpr BranchProbability getOne()
static uint32_t getDenominator()
static constexpr BranchProbability getUnknown()
static constexpr BranchProbability getZero()
uint32_t getNumerator() const
LLVM_ABI uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
CallingConv::ID getCallingConv() const
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Value * getCalledOperand() const
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
bool isConvergent() const
Determine if the invoke is convergent.
FunctionType * getFunctionType() const
unsigned arg_size() const
AttributeList getAttributes() const
Return the attributes for this call.
LLVM_ABI bool isTailCall() const
Tests if this call site is marked as a tail call.
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Conditional Branch instruction.
Class for constant bytes.
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
A constant value that is initialized with an expression using other constant values.
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
A signed pointer, in the ptrauth sense.
uint64_t getZExtValue() const
Constant Vector Declarations.
This is an important base class in LLVM.
This is the common base class for constrained floating point intrinsics.
LLVM_ABI std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
LLVM_ABI unsigned getNonMetadataArgCount() const
LLVM_ABI bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static LLVM_ABI void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static LLVM_ABI DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static LLVM_ABI std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
LLVM_ABI uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static LLVM_ABI std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static LLVM_ABI const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static LLVM_ABI DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static LLVM_ABI DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
Base class for variables.
LLVM_ABI std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
A parsed version of the target data layout string in and methods for querying it.
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
LocationType getType() const
LLVM_ABI Value * getVariableLocationOp(unsigned OpIdx) const
DIExpression * getExpression() const
DILocalVariable * getVariable() const
LLVM_ABI iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
LLVM_ABI DILocation * getInlinedAt() const
iterator find(const_arg_type_t< KeyT > Val)
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT, true > const_iterator
void reserve(size_type NumEntries)
Grow the DenseMap so that it can contain at least NumEntries items before resizing again.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for inline asm reporting.
static constexpr ElementCount getFixed(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
constexpr bool isScalar() const
Exactly one element.
Lightweight error class with error context and mandatory checking.
Class representing an expression and its matching format.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
bool allowReassoc() const
Flag queries.
An instruction for ordering other memory operations.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
MachineBasicBlock * getMBB(const BasicBlock *BB) const
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
MachineBasicBlock * MBB
MBB - The current block.
Class to represent function types.
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Type * getParamType(unsigned i) const
Parameter type accessors.
Type * getReturnType() const
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasParamAttribute(unsigned ArgNo, Attribute::AttrKind Kind) const
check if an attributes is in the list of attributes.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Constant * getPersonalityFn() const
Get the personality function associated with this function.
AttributeList getAttributes() const
Return the attribute list for this Function.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Garbage collection metadata for a single function.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
bool hasDLLImportStorageClass() const
Module * getParent()
Get the module that this global value is contained inside of...
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
void setMemConstraint(ConstraintCode C)
setMemConstraint - Augment an existing flag with the constraint code for a memory constraint.
This instruction inserts a struct field of array element value into an aggregate value.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
LLVM_ABI AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
@ OB_clang_arc_attachedcall
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
A helper class to return the specified delimiter string after the first invocation of operator String...
An instruction for reading from memory.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
static LocationSize upperBound(uint64_t Value)
bool usesPerInvokeEHLabels(ExceptionHandling EH) const
Returns true if the exception tables reference the per-invoke EH labels emitted around invokes.
LLVM_ABI MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
LLVM_ABI MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
const MCAsmInfo & getAsmInfo() const
unsigned getID() const
getID() - Return the register class ID number.
bool isAllocatable() const
isAllocatable - Return true if this register class may be used to create virtual registers.
const MCPhysReg * iterator
iterator begin() const
begin/end - Return all of the registers in this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
const MDOperand & getOperand(unsigned I) const
LLVM_ABI StringRef getString() const
@ INVALID_SIMPLE_VALUE_TYPE
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
std::string getString() const
Return the value type as a string, e.g. i32.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
succ_iterator succ_begin()
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
SmallVectorImpl< MachineBasicBlock * >::iterator succ_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHContTarget(bool V=true)
Indicates if this is a target of Windows EH Continuation Guard.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
MachineInstrBundleIterator< MachineInstr > iterator
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
void setIsAliasedObjectIndex(int ObjectIdx, bool IsAliased)
Set "maybe pointed to by an LLVM IR value" for an object.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
bool hasEHFunclets() const
void setHasEHContTarget(bool V)
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI MCRegister getLiveInPhysReg(Register VReg) const
getLiveInPhysReg - If VReg is a live-in virtual register, return the corresponding live-in physical r...
An SDNode that represents everything that will be needed to construct a MachineInstr.
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
bool contains(const KeyT &Key) const
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
A Module instance is used to store all the information related to an LLVM module.
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
static PointerType * getUnqual(LLVMContext &C)
This constructs an opaque pointer to an object in the default address space (address space zero).
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(Register VReg)
static SDDbgOperand fromConst(const Value *Const)
static SDDbgOperand fromGlobalAddr(const GlobalValue *GV)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const CondBrInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
DenseMap< const Constant *, Register > ConstantsOut
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
SDValue lowerStartEH(SDValue Chain, const BasicBlock *EHPadBB, MCSymbol *&BeginLabel)
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr, const TargetLowering::PtrAuthInfo *PAI=nullptr)
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
const TargetTransformInfo * TTI
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
SDValue lowerNoFPClassToAssertNoFPClass(SelectionDAG &DAG, const Instruction &I, SDValue Op)
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void setValueToPoison(const Value *V, const SDLoc &dl)
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, Register Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
bool canTailCall(const CallBase &CB) const
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void CopyValueToVirtualRegister(const Value *V, Register Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
void init(GCFunctionInfo *gfi, BatchAAResults *BatchAA, AssumptionCache *AC, const TargetLibraryInfo *li, const TargetTransformInfo &TTI)
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
DebugLoc getCurDebugLoc() const
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
SDLoc getCurSDLoc() const
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
SDValue getFPOperationRoot(fp::ExceptionBehavior EB)
Return the current virtual root of the Selection DAG, flushing PendingConstrainedFP or PendingConstra...
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
void LowerCallSiteWithPtrAuthBundle(const CallBase &CB, const BasicBlock *EHPadBB)
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
void LowerDeoptimizingReturn()
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
MachineRegisterInfo * RegInfo
std::unique_ptr< SwiftErrorValueTracking > SwiftError
virtual void emitFunctionEntryCode()
std::unique_ptr< SelectionDAGBuilder > SDB
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemccpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue C, SDValue Size, const CallInst *CI) const
Emit target-specific code that performs a memccpy, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, const CallInst *CI) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrstr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, const CallInst *CI) const
Emit target-specific code that performs a strstr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo, const CallInst *CI) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, const CallInst *CI) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy, const CallInst *CI) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
LLVM_ABI bool shouldOptForSize() const
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
LLVM_ABI void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
LLVM_ABI SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
SDValue getTargetFrameIndex(int FI, EVT VT)
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
LLVM_ABI SDValue getBasicBlock(MachineBasicBlock *MBB)
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
LLVM_ABI SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
MachineBasicBlock * getParentMBB()
bool shouldEmitFunctionBasedCheckStackProtector() const
An instruction for storing to memory.
Represent a constant reference to a string, i.e.
constexpr bool empty() const
Check if the string is empty.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Provides information about what library functions are available for the current target.
virtual Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Returns the desired alignment for ByVal or InAlloca aggregate function arguments in the caller parame...
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
virtual bool isAtomicAlignmentSupported(Align Alignment, uint64_t SizeInBytes) const
Return true if the target supports an atomic access of SizeInBytes bytes at the given Alignment.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
virtual Register getExceptionSelectorRegister(ExceptionHandling EH, const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual bool useStackGuardMixFP() const
If this function returns true, stack protection checks should mix the frame pointer (or whichever poi...
MVT getRegisterType(LLVMContext &Context, EVT VT) const
Return the type of registers that this ValueType will eventually require.
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr, CodeGenOptLevel OptLevel=CodeGenOptLevel::Default) const
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
MachineMemOperand::Flags getVPIntrinsicMemOperandFlags(const VPIntrinsic &VPIntrin) const
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
virtual Value * getSDagStackGuard(const Module &M, const LibcallLoweringInfo &Libcalls) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
virtual Register getExceptionPointerRegister(ExceptionHandling EH, const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_UDIV, ISD::VP_SDIV,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual SDValue emitStackGuardMixFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue Ptr, SDValue Val, SDValue Mask) const
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
SDValue annotateStackObjectPointer(SDValue Ptr, SelectionDAG &DAG, const SDLoc &DL, Align Alignment) const
Annotate a stack object pointer with known-bits assertions.
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &, const Type *RetTy) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
Primary interface to the complete machine description for the target machine.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
CodeModel::Model getCodeModel() const
Returns the code model.
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM_ABI bool isEmptyTy() const
Return true if this type is empty, that is, it has no elements or all of its elements are empty.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isTokenTy() const
Return true if this is 'token'.
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isVoidTy() const
Return true if this is 'void'.
Unconditional Branch instruction.
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
unsigned getNumOperands() const
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
This is the common base class for vector predication intrinsics.
static LLVM_ABI std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
LLVM_ABI MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
LLVMContext & getContext() const
All values hold a context through their type.
iterator_range< user_iterator > users()
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Base class of all SIMD vector types.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
const ParentTy * getParent() const
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ CONVERGENCECTRL_ANCHOR
The llvm.experimental.convergence.* intrinsics.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SET_FPENV
Sets the current floating-point environment.
@ LOOP_DEPENDENCE_RAW_MASK
@ MASK_BEFOREFIRST
Has one mask vector operand.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ COND_LOOP
COND_LOOP is a conditional branch to self, used for implementing efficient conditional traps.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ STACKADDRESS
STACKADDRESS - Represents the llvm.stackaddress intrinsic.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ RESET_FPENV
Set floating-point environment to default state.
@ ADD
Simple integer binary arithmetic operators.
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ VECTOR_FIND_LAST_ACTIVE
Finds the index of the last active mask element Operands: Mask.
@ FMODF
FMODF - Decomposes the operand into integral and fractional parts, each having the same type and sign...
@ FATAN2
FATAN2 - atan2, inspired by libm.
@ FSINCOSPI
FSINCOSPI - Compute both the sine and cosine times pi more accurately than FSINCOS(pi*x),...
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ VECTOR_SHUFFLE_VAR
VECTOR_SHUFFLE_VAR(VEC, MASK) - Returns VEC permuted by MASK, an integer vector with the same number ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ FMULADD
FMULADD - Performs a * b + c, with, or without, intermediate rounding.
@ FPTRUNC_ROUND
FPTRUNC_ROUND - This corresponds to the fptrunc_round intrinsic.
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ CLMUL
Carry-less multiplication operations.
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ CONVERT_FROM_ARBITRARY_FP
CONVERT_FROM_ARBITRARY_FP - This operator converts from an arbitrary floating-point represented as an...
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
@ SET_ROUNDING
Set rounding mode.
@ CONVERGENCECTRL_GLUE
This does not correspond to any convergence control intrinsic.
@ SIGN_EXTEND
Conversion operators.
@ PREALLOCATED_SETUP
PREALLOCATED_SETUP - This has 2 operands: an input chain and a SRCVALUE with the preallocated call Va...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ BR
Control flow instructions. These all have token chains.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ VECREDUCE_FMAXIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM nodes do not propagate NaNs and order signed zeroes using the llvm....
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SSUBO
Same for subtraction.
@ PREALLOCATED_ARG
PREALLOCATED_ARG - This has 3 operands: an input chain, a SRCVALUE with the preallocated call Value,...
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ GET_FPENV
Gets the current floating-point environment.
@ SHL
Shift and rotation operations.
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
@ PtrAuthGlobalAddress
A ptrauth constant.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ PATCHPOINT
The llvm.experimental.patchpoint.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ MASKED_UDIV
Masked vector arithmetic that returns poison on disabled lanes.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ PCMARKER
PCMARKER - This corresponds to the pcmarker intrinsic.
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ RELOC_NONE
Issue a no-op relocation against a given symbol at the current location.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ VECTOR_MATCH
VECTOR_MATCH - this corresponds to the llvm.experimental.vector.match intrinsic.
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ STACKMAP
The llvm.experimental.stackmap intrinsic.
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ CLEAR_CACHE
llvm.clear_cache intrinsic Operands: Input Chain, Start Addres, End Address Outputs: Output Chain
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ CONVERT_TO_ARBITRARY_FP
CONVERT_TO_ARBITRARY_FP - Converts a native FP value to an arbitrary floating-point format,...
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_REPEAT
VECTOR_REPEAT(FIXED_LENGTH_VECTOR) Repeatedly copies the elements of the source fixed-length vector t...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
@ LOOP_DEPENDENCE_WAR_MASK
The llvm.loop.dependence.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
Flag
These should be considered private to the implementation of the MCInstrDesc class.
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
auto m_VScale()
Matches a call to llvm.vscale().
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
LLVM_ABI void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
std::vector< CaseCluster > CaseClusterVector
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
SmallVector< SwitchWorkListItem, 4 > SwitchWorkList
CaseClusterVector::iterator CaseClusterIt
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
ExceptionBehavior
Exception behavior used for floating point operations.
@ ebStrict
This corresponds to "fpexcept.strict".
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
@ ebIgnore
This corresponds to "fpexcept.ignore".
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract_or_null(Y &&MD)
Extract a Value from Metadata, allowing null.
NodeAddr< FuncNode * > Func
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Type * getValueType(Value *V, bool ReVec, bool LookThroughCmp)
Returns the "element type" of the given value/instruction V.
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
LLVM_ABI ISD::NodeType getAtomicRMWOpcode(AtomicRMWInst::BinOp Op)
Return the atomic node opcode for an atomicrmw operation.
LLVM_ABI bool canDescribeGlobalAddressInLocationList(const MachineFunction &MF)
Test if the debug info for MF can name a describable global address in a location list too,...
LLVM_ABI ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
LLVM_ABI cl::opt< bool > ProfcheckDisableMetadataFixes
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
SDValue peekThroughFreeze(SDValue V)
Return the non-frozen source operand of V if it exists.
LLVM_ABI void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
RelativeUniformCounterPtr Values
@ Unknown
Not known to have no common set bits.
LLVM_ABI void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs=nullptr, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
LLVM_ABI void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
LLVM_ABI bool isExceptionPointerAndSelectorType(Type *Ty)
Return true if landingpad result type Ty is a struct of an exception pointer (pointer or integer) and...
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
static ConstantRange getRange(Value *Op, SCCPSolver &Solver, const SmallPtrSetImpl< Value * > &InsertedValues)
Helper for getting ranges from Solver.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
auto cast_or_null(const Y &Val)
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
gep_type_iterator gep_type_end(const User *GEP)
LLVM_ABI LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
constexpr auto equal_to(T &&Arg)
Functor variant of std::equal_to that can be used as a UnaryPredicate in functional algorithms like a...
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
detail::concat_range< ValueT, RangeTs... > concat(RangeTs &&...Ranges)
Returns a concatenated range across two or more ranges.
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
LLVM_ABI void ComputeValueTypes(const DataLayout &DL, Type *Ty, SmallVectorImpl< Type * > &Types, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
Given an LLVM IR type, compute non-aggregate subtypes.
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
FPOp
The floating-point operations supported by the folding helpers.
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
void sort(IteratorTy Start, IteratorTy End)
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
generic_gep_type_iterator<> gep_type_iterator
auto succ_size(const MachineBasicBlock *BB)
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
LLVM_ABI ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
LLVM_ABI MDNode * getMemCacheHint(const Instruction &I, unsigned OperandNo=0)
Return the hint payload for operand OperandNo of I, or nullptr if absent.
LLVM_ABI EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Sub
Subtraction of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
LLVM_ABI bool isOnlyUsedInZeroEqualityComparison(const Instruction *CtxI)
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
LLVM_ABI bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
DWARFExpression::Operation Op
@ Dynamic
Denotes mode unknown at compile time.
LLVM_ABI ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
LLVM_ABI const GlobalValue * getDescribableGlobalAddress(const Constant *C, int64_t &Offset, const MachineFunction &MF)
If C is the address of a global, possibly displaced by a constant, return that global and set Offset ...
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
LLVM_ABI std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
LLVM_ABI GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
LLVM_ABI Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
LLVM_ABI unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
@ End
Run the machine verifier once, at the end of the pipeline.
LLVM_ABI SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
MCRegisterClass TargetRegisterClass
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isRISCVVectorTuple() const
Return true if this is a vector value type.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
void setPointerAddrSpace(unsigned AS)
void setOrigAlign(Align A)
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
A lightweight accessor for an operand bundle meant to be passed around by value.
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< std::pair< Register, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
bool isABIMangled() const
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< Register, 4 > Regs
This list holds the registers assigned to the values.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
void setUnpredictable(bool b)
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
A MapVector that performs no allocations if smaller than a certain size.
MachineBasicBlock * Default
BranchProbability DefaultProb
MachineBasicBlock * Parent
bool FallthroughUnreachable
MachineBasicBlock * ThisBB
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
BranchProbability TrueProb
BranchProbability FalseProb
MachineBasicBlock * TrueBB
MachineBasicBlock * FalseBB
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
Register Reg
The virtual register containing the index of the jump table entry to jump to.
MachineBasicBlock * Default
The MBB of the default bb, which is a successor of the range check MBB.
unsigned JTI
The JumpTableIndex for this jump table in the function.
MachineBasicBlock * MBB
The MBB into which to emit the code for the indirect jump.
std::optional< SDLoc > SL
The debug location of the instruction this JumpTable was produced from.
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setDeactivationSymbol(GlobalValue *Sym)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
bool IsPostTypeLegalization
SmallVector< SDValue, 4 > InVals
Type * OrigRetTy
Original unlegalized return type.
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
Type * RetTy
Same as OrigRetTy, or partially legalized for soft float libcalls.
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setPtrAuth(PtrAuthInfo Value)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
This structure contains the information necessary for lowering pointer-authenticating indirect calls.
LLVM_ABI void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)