79#include "llvm/IR/IntrinsicsWebAssembly.h"
116#define DEBUG_TYPE "isel"
117#define ISEL_DUMP_DEBUG_TYPE DEBUG_TYPE "-dump"
119STATISTIC(NumFastIselFailures,
"Number of instructions fast isel failed on");
120STATISTIC(NumFastIselSuccess,
"Number of instructions fast isel selected");
121STATISTIC(NumFastIselBlocks,
"Number of blocks selected entirely by fast isel");
122STATISTIC(NumDAGBlocks,
"Number of blocks selected using DAG");
123STATISTIC(NumDAGIselRetries,
"Number of times dag isel has to try another path");
124STATISTIC(NumEntryBlocks,
"Number of entry blocks encountered");
126 "Number of entry blocks where fast isel failed to lower arguments");
130 cl::desc(
"Enable abort calls when \"fast\" instruction selection "
131 "fails to lower an instruction: 0 disable the abort, 1 will "
132 "abort but for args, calls and terminators, 2 will also "
133 "abort for argument lowering, and 3 will never fallback "
134 "to SelectionDAG."));
138 cl::desc(
"Emit a diagnostic when \"fast\" instruction selection "
139 "falls back to SelectionDAG."));
143 cl::desc(
"use Machine Branch Probability Info"),
149 cl::desc(
"Only display the basic block whose name "
150 "matches this for all view-*-dags options"));
153 cl::desc(
"Pop up a window to show dags before the first "
154 "dag combine pass"));
157 cl::desc(
"Pop up a window to show dags before legalize types"));
160 cl::desc(
"Pop up a window to show dags before the post "
161 "legalize types dag combine pass"));
164 cl::desc(
"Pop up a window to show dags before legalize"));
167 cl::desc(
"Pop up a window to show dags before the second "
168 "dag combine pass"));
171 cl::desc(
"Pop up a window to show isel dags as they are selected"));
174 cl::desc(
"Pop up a window to show sched dags as they are processed"));
177 cl::desc(
"Pop up a window to show SUnit dags after they are processed"));
186#define ISEL_DUMP(X) \
188 if (llvm::DebugFlag && \
189 (isCurrentDebugType(DEBUG_TYPE) || \
190 (isCurrentDebugType(ISEL_DUMP_DEBUG_TYPE) && MatchFilterFuncName))) { \
195#define ISEL_DUMP(X) do { } while (false)
215 cl::desc(
"Instruction schedulers available (before register"
228 return Arg.hasAttribute(Attribute::AttrKind::SwiftAsync);
247 if (NewOptLevel != SavedOptLevel) {
250 LLVM_DEBUG(
dbgs() <<
"\nChanging optimization level for Function "
252 LLVM_DEBUG(
dbgs() <<
"\tBefore: -O" <<
static_cast<int>(SavedOptLevel)
253 <<
" ; After: -O" <<
static_cast<int>(NewOptLevel)
261 dbgs() <<
"\tFastISel is "
269 LLVM_DEBUG(
dbgs() <<
"\nRestoring optimization level for Function "
272 <<
" ; After: -O" <<
static_cast<int>(SavedOptLevel) <<
"\n");
288 if (
auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
289 return SchedulerCtor(IS, OptLevel);
293 (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||
307 "Unknown sched type!");
326 dbgs() <<
"If a target marks an instruction with "
327 "'usesCustomInserter', it must implement "
328 "TargetLowering::EmitInstrWithCustomInserter!\n";
336 "If a target marks an instruction with 'hasPostISelHook', "
337 "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
388 if (!TT.isWindowsMSVCEnvironment())
396 if (
I.getType()->isFPOrFPVectorTy()) {
400 for (
const auto &
Op :
I.operands()) {
401 if (
Op->getType()->isFPOrFPVectorTy()) {
416 "-fast-isel-abort > 0 requires -fast-isel");
448 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(Fn);
449 GFI = Fn.
hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) :
nullptr;
450 ORE = std::make_unique<OptimizationRemarkEmitter>(&Fn);
451 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(mf.
getFunction());
452 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
455 BFI = &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI();
459 FnVarLocs = getAnalysis<AssignmentTrackingAnalysis>().getResults();
464 if (
auto *UAPass = getAnalysisIfAvailable<UniformityInfoWrapperPass>())
465 UA = &UAPass->getUniformityInfo();
476 FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
481 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
502 if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))
516 SelectAllBasicBlocks(Fn);
544 MRI.constrainRegClass(To,
MRI.getRegClass(
From));
550 if (!
MRI.use_empty(To))
570 if (Term !=
MBB.
end() && Term->isReturn()) {
579 if (!
FuncInfo->ArgDbgValues.empty())
585 for (
unsigned i = 0, e =
FuncInfo->ArgDbgValues.size(); i != e; ++i) {
587 assert(
MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
588 "Function parameters should not be described by DBG_VALUE_LIST.");
589 bool hasFI =
MI->getDebugOperand(0).isFI();
591 hasFI ?
TRI.getFrameRegister(*
MF) :
MI->getDebugOperand(0).getReg();
592 if (Reg.isPhysical())
599 Def->getParent()->insert(std::next(InsertPos),
MI);
611 if (LDI != LiveInMap.
end()) {
612 assert(!hasFI &&
"There's no handling of frame pointer updating here yet "
616 const MDNode *Variable =
MI->getDebugVariable();
617 const MDNode *Expr =
MI->getDebugExpression();
619 bool IsIndirect =
MI->isIndirectDebugValue();
621 assert(
MI->getDebugOffset().getImm() == 0 &&
622 "DBG_VALUE with nonzero offset");
623 assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(
DL) &&
624 "Expected inlined-at fields to agree");
625 assert(
MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
626 "Didn't expect to see a DBG_VALUE_LIST here");
629 IsIndirect, LDI->second, Variable, Expr);
641 CopyUseMI =
UseMI;
continue;
644 CopyUseMI =
nullptr;
break;
647 TRI.getRegSizeInBits(LDI->second,
MRI) ==
667 for (
const auto &
MBB : *
MF) {
671 for (
const auto &
MI :
MBB) {
674 MI.isStackAligningInlineAsm()) {
677 if (
MI.isInlineAsm()) {
693 ISEL_DUMP(
dbgs() <<
"*** MachineFunction at end of ISel ***\n");
705 if (!R.getLocation().isValid() || ShouldAbort)
706 R << (
" (in function: " + MF.
getName() +
")").str();
728 SDB->visitDbgInfo(*
I);
733 HadTailCall =
SDB->HasTailCall;
734 SDB->resolveOrClearDbgInfo();
741void SelectionDAGISel::ComputeLiveOutVRegInfo() {
755 if (
Op.getValueType() == MVT::Other &&
Added.insert(
Op.getNode()).second)
762 unsigned DestReg = cast<RegisterSDNode>(
N->getOperand(1))->getReg();
768 EVT SrcVT = Src.getValueType();
774 FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);
775 }
while (!Worklist.
empty());
778void SelectionDAGISel::CodeGenAndEmitDAG() {
780 StringRef GroupDescription =
"Instruction Selection and Scheduling";
781 std::string BlockName;
782 bool MatchFilterBB =
false; (void)MatchFilterBB;
785 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*
FuncInfo->Fn);
794 FuncInfo->MBB->getBasicBlock()->getName());
871 ISEL_DUMP(
dbgs() <<
"\nOptimized type-legalized selection DAG: "
905 ISEL_DUMP(
dbgs() <<
"\nVector/type-legalized selection DAG: "
925 ISEL_DUMP(
dbgs() <<
"\nOptimized vector-legalized selection DAG: "
976 ComputeLiveOutVRegInfo();
986 DoInstructionSelection();
1022 if (FirstMBB != LastMBB)
1023 SDB->UpdateSplitBlock(FirstMBB, LastMBB);
1045 :
SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
1058 void NodeInserted(
SDNode *
N)
override {
1059 SDNode *CurNode = &*ISelPosition;
1060 if (
MDNode *MD = DAG.getPCSections(CurNode))
1061 DAG.addPCSections(
N, MD);
1062 if (
MDNode *MMRA = DAG.getMMRAMetadata(CurNode))
1063 DAG.addMMRAMetadata(
N, MMRA);
1093 while (!Nodes.
empty()) {
1095 for (
auto *U :
N->uses()) {
1096 auto UId = U->getNodeId();
1109 int InvalidId = -(
N->getNodeId() + 1);
1110 N->setNodeId(InvalidId);
1115 int Id =
N->getNodeId();
1121void SelectionDAGISel::DoInstructionSelection() {
1124 <<
FuncInfo->MBB->getName() <<
"'\n");
1142 ISelUpdater ISU(*
CurDAG, ISelPosition);
1149 SDNode *Node = &*--ISelPosition;
1153 if (Node->use_empty())
1160 while (!Nodes.
empty()) {
1177 "Node has already selected predecessor node");
1194 switch (
Node->getOpcode()) {
1203 ActionVT =
Node->getOperand(1).getValueType();
1206 ActionVT =
Node->getValueType(0);
1214 LLVM_DEBUG(
dbgs() <<
"\nISEL: Starting selection on root node: ";
1230 if (
const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {
1232 if (IID == Intrinsic::eh_exceptionpointer ||
1233 IID == Intrinsic::eh_exceptioncode)
1248 bool IsSingleCatchAllClause =
1253 bool IsCatchLongjmp = CPI->
arg_size() == 0;
1254 if (!IsSingleCatchAllClause && !IsCatchLongjmp) {
1256 bool IntrFound =
false;
1258 if (
const auto *Call = dyn_cast<IntrinsicInst>(U)) {
1260 if (IID == Intrinsic::wasm_landingpad_index) {
1261 Value *IndexArg = Call->getArgOperand(1);
1262 int Index = cast<ConstantInt>(IndexArg)->getZExtValue();
1269 assert(IntrFound &&
"wasm.landingpad.index intrinsic not found!");
1276bool SelectionDAGISel::PrepareEHLandingPad() {
1288 if (
const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->
getFirstNonPHI())) {
1293 assert(EHPhysReg &&
"target lacks exception pointer register");
1295 unsigned VReg =
FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
1297 TII->
get(TargetOpcode::COPY), VReg)
1315 if (
auto *RegMask =
TRI.getCustomEHPadPreservedMask(*
MF))
1319 if (
const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->
getFirstNonPHI()))
1357 TII->
get(TargetOpcode::EH_LABEL))
1366 TII->
get(TargetOpcode::EH_LABEL))
1377 return !
I->mayWriteToMemory() &&
1378 !
I->isTerminator() &&
1379 !isa<DbgInfoIntrinsic>(
I) &&
1391 auto ArgIt = FuncInfo.
ValueMap.find(Arg);
1392 if (ArgIt == FuncInfo.
ValueMap.end())
1394 Register ArgVReg = ArgIt->getSecond();
1398 if (VirtReg == ArgVReg) {
1402 LLVM_DEBUG(
dbgs() <<
"processDbgDeclare: setVariableDbgInfo Var=" << *Var
1403 <<
", Expr=" << *Expr <<
", MCRegister=" << PhysReg
1404 <<
", DbgLoc=" << DbgLoc <<
"\n");
1415 <<
" (bad address)\n");
1425 assert(Var &&
"Missing variable");
1426 assert(DbgLoc &&
"Missing location");
1436 int FI = std::numeric_limits<int>::max();
1437 if (
const auto *AI = dyn_cast<AllocaInst>(
Address)) {
1441 }
else if (
const auto *Arg = dyn_cast<Argument>(
Address))
1444 if (FI == std::numeric_limits<int>::max())
1447 if (
Offset.getBoolValue())
1451 LLVM_DEBUG(
dbgs() <<
"processDbgDeclare: setVariableDbgInfo Var=" << *Var
1452 <<
", Expr=" << *Expr <<
", FI=" << FI
1453 <<
", DbgLoc=" << DbgLoc <<
"\n");
1462 const auto *DI = dyn_cast<DbgDeclareInst>(&
I);
1464 DI->getVariable(), DI->getDebugLoc()))
1469 DVR.getExpression(), DVR.getVariable(),
1484 assert(!It->Values.hasArgList() &&
"Single loc variadic ops not supported");
1490void SelectionDAGISel::SelectAllBasicBlocks(
const Function &Fn) {
1520 ++NumFastIselFailLowerArguments;
1525 R <<
"FastISel didn't lower all arguments: "
1533 CodeGenAndEmitDAG();
1547 if (FastIS && Inserted)
1552 "expected AssignmentTrackingAnalysis pass results");
1562 bool AllPredsVisited =
true;
1564 if (!
FuncInfo->VisitedBBs.count(Pred)) {
1565 AllPredsVisited =
false;
1570 if (AllPredsVisited) {
1572 FuncInfo->ComputePHILiveOutRegInfo(&PN);
1575 FuncInfo->InvalidatePHILiveOutRegInfo(&PN);
1578 FuncInfo->VisitedBBs.insert(LLVMBB);
1594 FuncInfo->ExceptionPointerVirtReg = 0;
1595 FuncInfo->ExceptionSelectorVirtReg = 0;
1597 if (!PrepareEHLandingPad())
1605 unsigned NumFastIselRemaining = std::distance(Begin,
End);
1611 for (; BI != Begin; --BI) {
1617 --NumFastIselRemaining;
1628 --NumFastIselRemaining;
1629 ++NumFastIselSuccess;
1636 while (BeforeInst != &*Begin) {
1641 if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1646 <<
"FastISel folded load: " << *BeforeInst <<
"\n");
1649 --NumFastIselRemaining;
1650 ++NumFastIselSuccess;
1662 if (isa<CallInst>(Inst) && !isa<GCStatepointInst>(Inst) &&
1663 !isa<GCRelocateInst>(Inst) && !isa<GCResultInst>(Inst)) {
1667 R <<
"FastISel missed call";
1670 std::string InstStrStorage;
1674 R <<
": " << InstStr.str();
1679 if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() &&
1680 !Inst->use_empty()) {
1686 bool HadTailCall =
false;
1688 SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);
1700 unsigned RemainingNow = std::distance(Begin, BI);
1701 NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1702 NumFastIselRemaining = RemainingNow;
1707 Inst->getDebugLoc(), LLVMBB);
1710 if (Inst->isTerminator()) {
1712 R <<
"FastISel missed terminator";
1716 R <<
"FastISel missed";
1720 std::string InstStrStorage;
1723 R <<
": " << InstStr.str();
1728 NumFastIselFailures += NumFastIselRemaining;
1736 bool FunctionBasedInstrumentation =
1738 SDB->SPDescriptor.initialize(LLVMBB,
FuncInfo->MBBMap[LLVMBB],
1739 FunctionBasedInstrumentation);
1745 ++NumFastIselBlocks;
1752 SelectBasicBlock(Begin, BI, HadTailCall);
1764 FuncInfo->PHINodesToUpdate.clear();
1770 reportIPToStateForBlocks(
MF);
1777 SDB->clearDanglingDebugInfo();
1778 SDB->SPDescriptor.resetPerFunctionState();
1782SelectionDAGISel::FinishBasicBlock() {
1784 <<
FuncInfo->PHINodesToUpdate.size() <<
"\n";
1785 for (
unsigned i = 0, e =
FuncInfo->PHINodesToUpdate.size(); i != e;
1787 <<
"Node " << i <<
" : (" <<
FuncInfo->PHINodesToUpdate[i].first
1788 <<
", " <<
FuncInfo->PHINodesToUpdate[i].second <<
")\n");
1792 for (
unsigned i = 0, e =
FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1795 "This is not a machine PHI node that we are updating!");
1796 if (!
FuncInfo->MBB->isSuccessor(
PHI->getParent()))
1802 if (
SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
1811 SDB->visitSPDescriptorParent(
SDB->SPDescriptor, ParentMBB);
1814 CodeGenAndEmitDAG();
1817 SDB->SPDescriptor.resetPerBBState();
1818 }
else if (
SDB->SPDescriptor.shouldEmitStackProtector()) {
1832 SuccessMBB->
splice(SuccessMBB->
end(), ParentMBB,
1839 SDB->visitSPDescriptorParent(
SDB->SPDescriptor, ParentMBB);
1842 CodeGenAndEmitDAG();
1846 if (FailureMBB->
empty()) {
1849 SDB->visitSPDescriptorFailure(
SDB->SPDescriptor);
1852 CodeGenAndEmitDAG();
1856 SDB->SPDescriptor.resetPerBBState();
1860 for (
auto &BTB :
SDB->SL->BitTestCases) {
1870 CodeGenAndEmitDAG();
1874 for (
unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
1875 UnhandledProb -= BTB.Cases[
j].ExtraProb;
1890 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1893 NextMBB = BTB.Cases[
j + 1].TargetBB;
1894 }
else if (j + 1 == ej) {
1896 NextMBB = BTB.Default;
1899 NextMBB = BTB.Cases[
j + 1].ThisBB;
1902 SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
1907 CodeGenAndEmitDAG();
1909 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1911 BTB.Cases.pop_back();
1917 for (
const std::pair<MachineInstr *, unsigned> &
P :
1922 "This is not a machine PHI node that we are updating!");
1925 if (PHIBB == BTB.Default) {
1926 PHI.addReg(
P.second).addMBB(BTB.Parent);
1927 if (!BTB.ContiguousRange) {
1928 PHI.addReg(
P.second).addMBB(BTB.Cases.back().ThisBB);
1935 PHI.addReg(
P.second).addMBB(cBB);
1939 SDB->SL->BitTestCases.clear();
1944 for (
unsigned i = 0, e =
SDB->SL->JTCases.size(); i != e; ++i) {
1946 if (!
SDB->SL->JTCases[i].first.Emitted) {
1948 FuncInfo->MBB =
SDB->SL->JTCases[i].first.HeaderBB;
1951 SDB->visitJumpTableHeader(
SDB->SL->JTCases[i].second,
1955 CodeGenAndEmitDAG();
1962 SDB->visitJumpTable(
SDB->SL->JTCases[i].second);
1965 CodeGenAndEmitDAG();
1968 for (
unsigned pi = 0, pe =
FuncInfo->PHINodesToUpdate.size();
1973 "This is not a machine PHI node that we are updating!");
1975 if (PHIBB ==
SDB->SL->JTCases[i].second.Default)
1977 .addMBB(
SDB->SL->JTCases[i].first.HeaderBB);
1979 if (
FuncInfo->MBB->isSuccessor(PHIBB))
1983 SDB->SL->JTCases.clear();
1987 for (
unsigned i = 0, e =
SDB->SL->SwitchCases.size(); i != e; ++i) {
1995 if (
SDB->SL->SwitchCases[i].TrueBB !=
SDB->SL->SwitchCases[i].FalseBB)
2002 CodeGenAndEmitDAG();
2012 for (
unsigned i = 0, e = Succs.
size(); i != e; ++i) {
2023 for (
unsigned pn = 0; ; ++pn) {
2025 "Didn't find PHI entry!");
2026 if (
FuncInfo->PHINodesToUpdate[pn].first ==
PHI) {
2027 PHI.addReg(
FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
2035 SDB->SL->SwitchCases.clear();
2056 int64_t DesiredMaskS)
const {
2057 const APInt &ActualMask =
RHS->getAPIntValue();
2058 const APInt &DesiredMask =
APInt(
LHS.getValueSizeInBits(), DesiredMaskS);
2061 if (ActualMask == DesiredMask)
2070 APInt NeededMask = DesiredMask & ~ActualMask;
2085 int64_t DesiredMaskS)
const {
2086 const APInt &ActualMask =
RHS->getAPIntValue();
2087 const APInt &DesiredMask =
APInt(
LHS.getValueSizeInBits(), DesiredMaskS);
2090 if (ActualMask == DesiredMask)
2099 APInt NeededMask = DesiredMask & ~ActualMask;
2116 std::vector<SDValue> InOps;
2125 if (InOps[e-1].getValueType() == MVT::Glue)
2130 if (!Flags.isMemKind() && !Flags.isFuncKind()) {
2132 Ops.insert(Ops.end(), InOps.begin() + i,
2133 InOps.begin() + i + Flags.getNumOperandRegisters() + 1);
2134 i += Flags.getNumOperandRegisters() + 1;
2136 assert(Flags.getNumOperandRegisters() == 1 &&
2137 "Memory operand with multiple values?");
2139 unsigned TiedToOperand;
2140 if (Flags.isUseOperandTiedToDef(TiedToOperand)) {
2144 for (; TiedToOperand; --TiedToOperand) {
2145 CurOp += Flags.getNumOperandRegisters() + 1;
2151 std::vector<SDValue> SelOps;
2153 Flags.getMemoryConstraintID();
2162 Flags.setMemConstraint(ConstraintID);
2170 if (e != InOps.size())
2171 Ops.push_back(InOps.back());
2178 unsigned FlagResNo =
N->getNumValues()-1;
2181 if (
Use.getResNo() == FlagResNo)
2190 bool IgnoreChains) {
2199 Visited.
insert(ImmedUse);
2204 if ((
Op.getValueType() == MVT::Other && IgnoreChains) ||
N == Def)
2206 if (!Visited.
insert(
N).second)
2212 if (Root != ImmedUse) {
2216 if ((
Op.getValueType() == MVT::Other && IgnoreChains) ||
N == Def)
2218 if (!Visited.
insert(
N).second)
2233 return N.hasOneUse();
2240 bool IgnoreChains) {
2289 while (VT == MVT::Glue) {
2300 IgnoreChains =
false;
2306void SelectionDAGISel::Select_INLINEASM(
SDNode *
N) {
2309 std::vector<SDValue> Ops(
N->op_begin(),
N->op_end());
2312 const EVT VTs[] = {MVT::Other, MVT::Glue};
2319void SelectionDAGISel::Select_READ_REGISTER(
SDNode *
Op) {
2324 EVT VT =
Op->getValueType(0);
2330 Op->getOperand(0), dl, Reg,
Op->getValueType(0));
2336void SelectionDAGISel::Select_WRITE_REGISTER(
SDNode *
Op) {
2341 EVT VT =
Op->getOperand(2).getValueType();
2347 Op->getOperand(0), dl, Reg,
Op->getOperand(2));
2353void SelectionDAGISel::Select_UNDEF(
SDNode *
N) {
2357void SelectionDAGISel::Select_FREEZE(
SDNode *
N) {
2365void SelectionDAGISel::Select_ARITH_FENCE(
SDNode *
N) {
2370void SelectionDAGISel::Select_MEMBARRIER(
SDNode *
N) {
2375void SelectionDAGISel::Select_CONVERGENCECTRL_ANCHOR(
SDNode *
N) {
2377 N->getValueType(0));
2380void SelectionDAGISel::Select_CONVERGENCECTRL_ENTRY(
SDNode *
N) {
2382 N->getValueType(0));
2385void SelectionDAGISel::Select_CONVERGENCECTRL_LOOP(
SDNode *
N) {
2387 N->getValueType(0),
N->getOperand(0));
2408void SelectionDAGISel::Select_STACKMAP(
SDNode *
N) {
2410 auto *It =
N->op_begin();
2419 assert(
ID.getValueType() == MVT::i64);
2428 for (; It !=
N->op_end(); It++)
2429 pushStackMapLiveVariable(Ops, *It,
DL);
2438void SelectionDAGISel::Select_PATCHPOINT(
SDNode *
N) {
2440 auto *It =
N->op_begin();
2445 std::optional<SDValue> Glue;
2446 if (It->getValueType() == MVT::Glue)
2452 assert(
ID.getValueType() == MVT::i64);
2476 for (; It !=
N->op_end(); It++)
2477 pushStackMapLiveVariable(Ops, *It,
DL);
2482 if (Glue.has_value())
2492 assert(Val >= 128 &&
"Not a VBR");
2498 NextBits = MatcherTable[
Idx++];
2499 Val |= (NextBits&127) << Shift;
2501 }
while (NextBits & 128);
2506void SelectionDAGISel::Select_JUMP_TABLE_DEBUG_INFO(
SDNode *
N) {
2510 dl, MVT::i64,
true));
2515void SelectionDAGISel::UpdateChains(
2522 if (!ChainNodesMatched.
empty()) {
2524 "Matched input chains but didn't produce a chain");
2527 for (
unsigned i = 0, e = ChainNodesMatched.
size(); i != e; ++i) {
2528 SDNode *ChainNode = ChainNodesMatched[i];
2535 "Deleted node left in chain");
2539 if (ChainNode == NodeToMatch && isMorphNodeTo)
2548 std::replace(ChainNodesMatched.
begin(), ChainNodesMatched.
end(),
N,
2549 static_cast<SDNode *
>(
nullptr));
2555 if (ChainNode != NodeToMatch && ChainNode->
use_empty() &&
2561 if (!NowDeadNodes.
empty())
2580 unsigned int Max = 8192;
2583 if (ChainNodesMatched.
size() == 1)
2584 return ChainNodesMatched[0]->getOperand(0);
2588 std::function<void(
const SDValue)> AddChains = [&](
const SDValue V) {
2589 if (V.getValueType() != MVT::Other)
2593 if (!Visited.
insert(V.getNode()).second)
2596 for (
const SDValue &
Op : V->op_values())
2602 for (
auto *
N : ChainNodesMatched) {
2607 while (!Worklist.
empty())
2611 if (InputChains.
size() == 0)
2621 for (
auto *
N : ChainNodesMatched)
2626 if (InputChains.
size() == 1)
2627 return InputChains[0];
2629 MVT::Other, InputChains);
2633SDNode *SelectionDAGISel::
2642 int OldGlueResultNo = -1, OldChainResultNo = -1;
2644 unsigned NTMNumResults =
Node->getNumValues();
2645 if (
Node->getValueType(NTMNumResults-1) == MVT::Glue) {
2646 OldGlueResultNo = NTMNumResults-1;
2647 if (NTMNumResults != 1 &&
2648 Node->getValueType(NTMNumResults-2) == MVT::Other)
2649 OldChainResultNo = NTMNumResults-2;
2650 }
else if (
Node->getValueType(NTMNumResults-1) == MVT::Other)
2651 OldChainResultNo = NTMNumResults-1;
2669 static_cast<unsigned>(OldGlueResultNo) != ResNumResults - 1)
2671 SDValue(Res, ResNumResults - 1));
2677 if ((EmitNodeInfo &
OPFL_Chain) && OldChainResultNo != -1 &&
2678 static_cast<unsigned>(OldChainResultNo) != ResNumResults - 1)
2680 SDValue(Res, ResNumResults - 1));
2698 unsigned RecNo = MatcherTable[MatcherIndex++];
2699 assert(RecNo < RecordedNodes.size() &&
"Invalid CheckSame");
2700 return N == RecordedNodes[RecNo].first;
2705 const unsigned char *MatcherTable,
unsigned &MatcherIndex,
SDValue N,
2708 if (ChildNo >=
N.getNumOperands())
2710 return ::CheckSame(MatcherTable, MatcherIndex,
N.getOperand(ChildNo),
2718 bool TwoBytePredNo =
2722 ? MatcherTable[MatcherIndex++]
2725 PredNo |= MatcherTable[MatcherIndex++] << 8;
2735 ? MatcherTable[MatcherIndex++]
2743 uint16_t Opc = MatcherTable[MatcherIndex++];
2744 Opc |=
static_cast<uint16_t>(MatcherTable[MatcherIndex++]) << 8;
2745 return N->getOpcode() == Opc;
2752 if (
N.getValueType() == VT)
2762 if (ChildNo >=
N.getNumOperands())
2764 return ::CheckType(VT,
N.getOperand(ChildNo), TLI,
DL);
2770 return cast<CondCodeSDNode>(
N)->get() ==
2777 if (2 >=
N.getNumOperands())
2779 return ::CheckCondCode(MatcherTable, MatcherIndex,
N.getOperand(2));
2787 if (cast<VTSDNode>(
N)->getVT() == VT)
2791 return VT == MVT::iPTR && cast<VTSDNode>(
N)->getVT() == TLI->
getPointerTy(
DL);
2808 int64_t Val = MatcherTable[MatcherIndex++];
2810 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2815 return C &&
C->getAPIntValue().trySExtValue() == Val;
2821 if (ChildNo >=
N.getNumOperands())
2823 return ::CheckInteger(MatcherTable, MatcherIndex,
N.getOperand(ChildNo));
2829 int64_t Val = MatcherTable[MatcherIndex++];
2831 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2833 if (
N->getOpcode() !=
ISD::AND)
return false;
2842 int64_t Val = MatcherTable[MatcherIndex++];
2844 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2846 if (
N->getOpcode() !=
ISD::OR)
return false;
2863 unsigned Opcode = Table[
Index++];
2923 unsigned Res = Table[
Index++];
2925 N.getValue(Res), SDISel.
TLI,
3011 unsigned NumRecordedNodes;
3014 unsigned NumMatchedMemRefs;
3017 SDValue InputChain, InputGlue;
3020 bool HasChainNodesMatched;
3037 :
SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
3038 RecordedNodes(
RN), MatchScopes(MS) {}
3046 if (!
E ||
E->isMachineOpcode())
3049 if (
N == *NodeToMatch)
3054 for (
auto &
I : RecordedNodes)
3055 if (
I.first.getNode() ==
N)
3058 for (
auto &
I : MatchScopes)
3059 for (
auto &J :
I.NodeStack)
3060 if (J.getNode() ==
N)
3068 const unsigned char *MatcherTable,
3069 unsigned TableSize) {
3108 Select_INLINEASM(NodeToMatch);
3111 Select_READ_REGISTER(NodeToMatch);
3114 Select_WRITE_REGISTER(NodeToMatch);
3117 Select_UNDEF(NodeToMatch);
3120 Select_FREEZE(NodeToMatch);
3123 Select_ARITH_FENCE(NodeToMatch);
3126 Select_MEMBARRIER(NodeToMatch);
3129 Select_STACKMAP(NodeToMatch);
3132 Select_PATCHPOINT(NodeToMatch);
3135 Select_JUMP_TABLE_DEBUG_INFO(NodeToMatch);
3138 Select_CONVERGENCECTRL_ANCHOR(NodeToMatch);
3141 Select_CONVERGENCECTRL_ENTRY(NodeToMatch);
3144 Select_CONVERGENCECTRL_LOOP(NodeToMatch);
3171 SDValue InputChain, InputGlue;
3185 unsigned MatcherIndex = 0;
3187 if (!OpcodeOffset.empty()) {
3189 if (
N.getOpcode() < OpcodeOffset.size())
3190 MatcherIndex = OpcodeOffset[
N.getOpcode()];
3191 LLVM_DEBUG(
dbgs() <<
" Initial Opcode index to " << MatcherIndex <<
"\n");
3200 unsigned CaseSize = MatcherTable[
Idx++];
3202 CaseSize =
GetVBR(CaseSize, MatcherTable,
Idx);
3203 if (CaseSize == 0)
break;
3207 Opc |=
static_cast<uint16_t>(MatcherTable[
Idx++]) << 8;
3208 if (Opc >= OpcodeOffset.size())
3209 OpcodeOffset.resize((Opc+1)*2);
3210 OpcodeOffset[Opc] =
Idx;
3215 if (
N.getOpcode() < OpcodeOffset.size())
3216 MatcherIndex = OpcodeOffset[
N.getOpcode()];
3220 assert(MatcherIndex < TableSize &&
"Invalid index");
3222 unsigned CurrentOpcodeIndex = MatcherIndex;
3236 unsigned NumToSkip = MatcherTable[MatcherIndex++];
3237 if (NumToSkip & 128)
3238 NumToSkip =
GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3240 if (NumToSkip == 0) {
3245 FailIndex = MatcherIndex+NumToSkip;
3247 unsigned MatcherIndexOfPredicate = MatcherIndex;
3248 (void)MatcherIndexOfPredicate;
3255 Result, *
this, RecordedNodes);
3260 dbgs() <<
" Skipped scope entry (due to false predicate) at "
3261 <<
"index " << MatcherIndexOfPredicate <<
", continuing at "
3262 << FailIndex <<
"\n");
3263 ++NumDAGIselRetries;
3267 MatcherIndex = FailIndex;
3271 if (FailIndex == 0)
break;
3275 MatchScope NewEntry;
3276 NewEntry.FailIndex = FailIndex;
3277 NewEntry.NodeStack.append(NodeStack.
begin(), NodeStack.
end());
3278 NewEntry.NumRecordedNodes = RecordedNodes.
size();
3279 NewEntry.NumMatchedMemRefs = MatchedMemRefs.
size();
3280 NewEntry.InputChain = InputChain;
3281 NewEntry.InputGlue = InputGlue;
3282 NewEntry.HasChainNodesMatched = !ChainNodesMatched.
empty();
3288 SDNode *Parent =
nullptr;
3289 if (NodeStack.
size() > 1)
3290 Parent = NodeStack[NodeStack.
size()-2].getNode();
3291 RecordedNodes.
push_back(std::make_pair(
N, Parent));
3300 if (ChildNo >=
N.getNumOperands())
3303 RecordedNodes.
push_back(std::make_pair(
N->getOperand(ChildNo),
3308 if (
auto *MN = dyn_cast<MemSDNode>(
N))
3309 MatchedMemRefs.
push_back(MN->getMemOperand());
3319 if (
N->getNumOperands() != 0 &&
3320 N->getOperand(
N->getNumOperands()-1).getValueType() == MVT::Glue)
3321 InputGlue =
N->getOperand(
N->getNumOperands()-1);
3325 unsigned ChildNo = MatcherTable[MatcherIndex++];
3326 if (ChildNo >=
N.getNumOperands())
3328 N =
N.getOperand(ChildNo);
3338 if (ChildNo >=
N.getNumOperands())
3340 N =
N.getOperand(ChildNo);
3356 assert(!NodeStack.
empty() &&
"Node stack imbalance!");
3357 N = NodeStack.
back();
3360 ? MatcherTable[MatcherIndex++]
3362 if (SiblingNo >=
N.getNumOperands())
3364 N =
N.getOperand(SiblingNo);
3371 assert(!NodeStack.
empty() &&
"Node stack imbalance!");
3372 N = NodeStack.
back();
3376 if (!
::CheckSame(MatcherTable, MatcherIndex,
N, RecordedNodes))
break;
3413 unsigned OpNum = MatcherTable[MatcherIndex++];
3416 for (
unsigned i = 0; i < OpNum; ++i)
3417 Operands.push_back(RecordedNodes[MatcherTable[MatcherIndex++]].first);
3419 unsigned PredNo = MatcherTable[MatcherIndex++];
3434 ? MatcherTable[MatcherIndex++]
3436 unsigned RecNo = MatcherTable[MatcherIndex++];
3437 assert(RecNo < RecordedNodes.
size() &&
"Invalid CheckComplexPat");
3441 std::unique_ptr<MatchStateUpdater> MSU;
3443 MSU.reset(
new MatchStateUpdater(*
CurDAG, &NodeToMatch, RecordedNodes,
3447 RecordedNodes[RecNo].first, CPNum,
3453 if (!
::CheckOpcode(MatcherTable, MatcherIndex,
N.getNode()))
break;
3476 unsigned Res = MatcherTable[MatcherIndex++];
3485 unsigned CurNodeOpcode =
N.getOpcode();
3486 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3490 CaseSize = MatcherTable[MatcherIndex++];
3492 CaseSize =
GetVBR(CaseSize, MatcherTable, MatcherIndex);
3493 if (CaseSize == 0)
break;
3495 uint16_t Opc = MatcherTable[MatcherIndex++];
3496 Opc |=
static_cast<uint16_t>(MatcherTable[MatcherIndex++]) << 8;
3499 if (CurNodeOpcode == Opc)
3503 MatcherIndex += CaseSize;
3507 if (CaseSize == 0)
break;
3510 LLVM_DEBUG(
dbgs() <<
" OpcodeSwitch from " << SwitchStart <<
" to "
3511 << MatcherIndex <<
"\n");
3516 MVT CurNodeVT =
N.getSimpleValueType();
3517 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3521 CaseSize = MatcherTable[MatcherIndex++];
3523 CaseSize =
GetVBR(CaseSize, MatcherTable, MatcherIndex);
3524 if (CaseSize == 0)
break;
3528 if (CaseVT == MVT::iPTR)
3532 if (CurNodeVT == CaseVT)
3536 MatcherIndex += CaseSize;
3540 if (CaseSize == 0)
break;
3544 <<
"] from " << SwitchStart <<
" to " << MatcherIndex
3614 if (!
::CheckOrImm(MatcherTable, MatcherIndex,
N, *
this))
break;
3626 assert(NodeStack.
size() != 1 &&
"No parent node");
3629 bool HasMultipleUses =
false;
3630 for (
unsigned i = 1, e = NodeStack.
size()-1; i != e; ++i) {
3631 unsigned NNonChainUses = 0;
3632 SDNode *NS = NodeStack[i].getNode();
3634 if (UI.getUse().getValueType() != MVT::Other)
3635 if (++NNonChainUses > 1) {
3636 HasMultipleUses =
true;
3639 if (HasMultipleUses)
break;
3641 if (HasMultipleUses)
break;
3680 int64_t Val = MatcherTable[MatcherIndex++];
3682 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
3685 RecordedNodes.
push_back(std::pair<SDValue, SDNode *>(
3704 unsigned RegNo = MatcherTable[MatcherIndex++];
3705 RecordedNodes.
push_back(std::pair<SDValue, SDNode *>(
3715 unsigned RegNo = MatcherTable[MatcherIndex++];
3716 RegNo |= MatcherTable[MatcherIndex++] << 8;
3717 RecordedNodes.
push_back(std::pair<SDValue, SDNode*>(
3733 ? MatcherTable[MatcherIndex++]
3735 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitConvertToTarget");
3736 SDValue Imm = RecordedNodes[RecNo].first;
3739 const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
3741 Imm.getValueType());
3743 const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
3745 Imm.getValueType());
3748 RecordedNodes.
push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3757 "EmitMergeInputChains should be the first chain producing node");
3759 "Should only have one EmitMergeInputChains per match");
3763 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitMergeInputChains");
3764 ChainNodesMatched.
push_back(RecordedNodes[RecNo].first.getNode());
3770 if (ChainNodesMatched.
back() != NodeToMatch &&
3771 !RecordedNodes[RecNo].first.hasOneUse()) {
3772 ChainNodesMatched.
clear();
3786 "EmitMergeInputChains should be the first chain producing node");
3793 unsigned NumChains = MatcherTable[MatcherIndex++];
3794 assert(NumChains != 0 &&
"Can't TF zero chains");
3797 "Should only have one EmitMergeInputChains per match");
3800 for (
unsigned i = 0; i != NumChains; ++i) {
3801 unsigned RecNo = MatcherTable[MatcherIndex++];
3802 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitMergeInputChains");
3803 ChainNodesMatched.
push_back(RecordedNodes[RecNo].first.getNode());
3809 if (ChainNodesMatched.
back() != NodeToMatch &&
3810 !RecordedNodes[RecNo].first.hasOneUse()) {
3811 ChainNodesMatched.
clear();
3817 if (ChainNodesMatched.
empty())
3842 : MatcherTable[MatcherIndex++];
3843 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitCopyToReg");
3844 unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3846 DestPhysReg |= MatcherTable[MatcherIndex++] << 8;
3852 DestPhysReg, RecordedNodes[RecNo].first,
3855 InputGlue = InputChain.
getValue(1);
3860 unsigned XFormNo = MatcherTable[MatcherIndex++];
3861 unsigned RecNo = MatcherTable[MatcherIndex++];
3862 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitNodeXForm");
3864 RecordedNodes.
push_back(std::pair<SDValue,SDNode*>(Res,
nullptr));
3870 unsigned index = MatcherTable[MatcherIndex++];
3871 index |= (MatcherTable[MatcherIndex++] << 8);
3903 uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3904 TargetOpc |=
static_cast<uint16_t>(MatcherTable[MatcherIndex++]) << 8;
3905 unsigned EmitNodeInfo;
3924 EmitNodeInfo = MatcherTable[MatcherIndex++];
3949 NumVTs = MatcherTable[MatcherIndex++];
3951 for (
unsigned i = 0; i != NumVTs; ++i) {
3954 if (VT == MVT::iPTR)
3967 if (VTs.
size() == 1)
3969 else if (VTs.
size() == 2)
3975 unsigned NumOps = MatcherTable[MatcherIndex++];
3977 for (
unsigned i = 0; i != NumOps; ++i) {
3978 unsigned RecNo = MatcherTable[MatcherIndex++];
3980 RecNo =
GetVBR(RecNo, MatcherTable, MatcherIndex);
3982 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitNode");
3983 Ops.
push_back(RecordedNodes[RecNo].first);
3990 FirstOpToCopy += (EmitNodeInfo &
OPFL_Chain) ? 1 : 0;
3992 "Invalid variadic node");
3995 for (
unsigned i = FirstOpToCopy, e = NodeToMatch->
getNumOperands();
3998 if (V.getValueType() == MVT::Glue)
break;
4013 bool MayRaiseFPException =
4020 bool IsMorphNodeTo =
4023 if (!IsMorphNodeTo) {
4030 for (
unsigned i = 0, e = VTs.
size(); i != e; ++i) {
4031 if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue)
break;
4037 "NodeToMatch was removed partway through selection");
4041 auto &Chain = ChainNodesMatched;
4043 "Chain node replaced during MorphNode");
4046 Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList,
4047 Ops, EmitNodeInfo));
4054 Flags.setNoFPExcept(
true);
4055 Res->setFlags(Flags);
4077 bool mayLoad = MCID.
mayLoad();
4084 if (MMO->isLoad()) {
4087 }
else if (MMO->isStore()) {
4099 <<
" Dropping mem operands\n";
4100 dbgs() <<
" " << (IsMorphNodeTo ?
"Morphed" :
"Created")
4105 if (IsMorphNodeTo) {
4107 UpdateChains(Res, InputChain, ChainNodesMatched,
true);
4117 unsigned NumResults = MatcherTable[MatcherIndex++];
4119 for (
unsigned i = 0; i != NumResults; ++i) {
4120 unsigned ResSlot = MatcherTable[MatcherIndex++];
4122 ResSlot =
GetVBR(ResSlot, MatcherTable, MatcherIndex);
4124 assert(ResSlot < RecordedNodes.
size() &&
"Invalid CompleteMatch");
4125 SDValue Res = RecordedNodes[ResSlot].first;
4127 assert(i < NodeToMatch->getNumValues() &&
4130 "Invalid number of results to complete!");
4136 "invalid replacement");
4141 UpdateChains(NodeToMatch, InputChain, ChainNodesMatched,
false);
4154 "Didn't replace all uses of the node?");
4164 LLVM_DEBUG(
dbgs() <<
" Match failed at index " << CurrentOpcodeIndex
4166 ++NumDAGIselRetries;
4168 if (MatchScopes.
empty()) {
4169 CannotYetSelect(NodeToMatch);
4175 MatchScope &LastScope = MatchScopes.
back();
4176 RecordedNodes.
resize(LastScope.NumRecordedNodes);
4178 NodeStack.
append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
4179 N = NodeStack.
back();
4181 if (LastScope.NumMatchedMemRefs != MatchedMemRefs.
size())
4182 MatchedMemRefs.
resize(LastScope.NumMatchedMemRefs);
4183 MatcherIndex = LastScope.FailIndex;
4187 InputChain = LastScope.InputChain;
4188 InputGlue = LastScope.InputGlue;
4189 if (!LastScope.HasChainNodesMatched)
4190 ChainNodesMatched.
clear();
4195 unsigned NumToSkip = MatcherTable[MatcherIndex++];
4196 if (NumToSkip & 128)
4197 NumToSkip =
GetVBR(NumToSkip, MatcherTable, MatcherIndex);
4201 if (NumToSkip != 0) {
4202 LastScope.FailIndex = MatcherIndex+NumToSkip;
4216 if (
N->isMachineOpcode()) {
4223 if (
N->isTargetOpcode())
4224 return N->isTargetStrictFPOpcode();
4225 return N->isStrictFPOpcode();
4230 auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
4235 if (
auto *FN = dyn_cast<FrameIndexSDNode>(
N->getOperand(0))) {
4238 int32_t Off =
C->getSExtValue();
4241 return (Off >= 0) && (((
A.value() - 1) & Off) ==
unsigned(Off));
4246void SelectionDAGISel::CannotYetSelect(
SDNode *
N) {
4249 Msg <<
"Cannot select: ";
4255 Msg <<
"\nIn function: " <<
MF->
getName();
4257 bool HasInputChain =
N->getOperand(0).getValueType() == MVT::Other;
4258 unsigned iid =
N->getConstantOperandVal(HasInputChain);
4259 if (iid < Intrinsic::num_intrinsics)
4262 Msg <<
"target intrinsic %" <<
TII->
getName(iid);
4264 Msg <<
"unknown intrinsic #" << iid;
unsigned const MachineRegisterInfo * MRI
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
MachineInstrBuilder & UseMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
amdgpu AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
Expand Atomic instructions
BlockVerifier::State From
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_ATTRIBUTE_ALWAYS_INLINE
LLVM_ATTRIBUTE_ALWAYS_INLINE - On compilers where we have a directive to do so, mark a method "always...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
This file defines the FastISel class.
mir Rename Register Operands
Machine Instruction Scheduler
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
static cl::opt< bool > ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed"))
static cl::opt< bool > ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the post " "legalize types dag combine pass"))
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckPatternPredicate(unsigned Opcode, const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel)
CheckPatternPredicate - Implements OP_CheckPatternPredicate.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes, unsigned ChildNo)
CheckChildSame - Implements OP_CheckChildXSame.
static uint64_t decodeSignRotatedValue(uint64_t V)
Decode a signed value stored with the sign bit in the LSB for dense VBR encoding.
static cl::opt< bool > ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected"))
static LLVM_ATTRIBUTE_ALWAYS_INLINE uint64_t GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx)
GetVBR - decode a vbr encoding whose top bit is set.
static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F, MachineModuleInfo &MMI)
static void reportFastISelFailure(MachineFunction &MF, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R, bool ShouldAbort)
static cl::opt< bool > ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the second " "dag combine pass"))
static RegisterScheduler defaultListDAGScheduler("default", "Best scheduler for the target", createDefaultScheduler)
static unsigned IsPredicateKnownToFail(const unsigned char *Table, unsigned Index, SDValue N, bool &Result, const SelectionDAGISel &SDISel, SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
IsPredicateKnownToFail - If we know how and can do so without pushing a scope, evaluate the current n...
static cl::opt< int > EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction selection " "fails to lower an instruction: 0 disable the abort, 1 will " "abort but for args, calls and terminators, 2 will also " "abort for argument lowering, and 3 will never fallback " "to SelectionDAG."))
static void mapWasmLandingPadIndex(MachineBasicBlock *MBB, const CatchPadInst *CPI)
static void processSingleLocVars(FunctionLoweringInfo &FuncInfo, FunctionVarLocs const *FnVarLocs)
Collect single location variable information generated with assignment tracking.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
static cl::opt< bool > UseMBPI("use-mbpi", cl::desc("use Machine Branch Probability Info"), cl::init(true), cl::Hidden)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildType(MVT::SimpleValueType VT, SDValue N, const TargetLowering *TLI, const DataLayout &DL, unsigned ChildNo)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
CheckSame - Implements OP_CheckSame.
static bool dontUseFastISelFor(const Function &Fn)
static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse, bool IgnoreChains)
findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path beyond "ImmedUse".
static cl::opt< bool > ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the first " "dag combine pass"))
static bool processIfEntryValueDbgDeclare(FunctionLoweringInfo &FuncInfo, const Value *Arg, DIExpression *Expr, DILocalVariable *Var, DebugLoc DbgLoc)
static cl::opt< bool > ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed"))
static void processDbgDeclares(FunctionLoweringInfo &FuncInfo)
Collect llvm.dbg.declare information.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(MVT::SimpleValueType VT, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
static cl::opt< bool > ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize"))
static cl::opt< bool > ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize types"))
static SDNode * findGlueUse(SDNode *N)
findGlueUse - Return use of MVT::Glue value produced by the specified SDNode.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckNodePredicate(unsigned Opcode, const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel, SDNode *N)
CheckNodePredicate - Implements OP_CheckNodePredicate.
static cl::opt< RegisterScheduler::FunctionPassCtor, false, RegisterPassParser< RegisterScheduler > > ISHeuristic("pre-RA-sched", cl::init(&createDefaultScheduler), cl::Hidden, cl::desc("Instruction schedulers available (before register" " allocation):"))
ISHeuristic command line option for instruction schedulers.
static cl::opt< bool > EnableFastISelFallbackReport("fast-isel-report-on-fallback", cl::Hidden, cl::desc("Emit a diagnostic when \"fast\" instruction selection " "falls back to SelectionDAG."))
static bool processDbgDeclare(FunctionLoweringInfo &FuncInfo, const Value *Address, DIExpression *Expr, DILocalVariable *Var, DebugLoc DbgLoc)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, unsigned ChildNo)
static cl::opt< std::string > FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, cl::desc("Only display the basic block whose name " "matches this for all view-*-dags options"))
static SDValue HandleMergeInputChains(SmallVectorImpl< SDNode * > &ChainNodesMatched, SelectionDAG *CurDAG)
HandleMergeInputChains - This implements the OPC_EmitMergeInputChains operation for when the pattern ...
static bool isFoldedOrDeadInstruction(const Instruction *I, const FunctionLoweringInfo &FuncInfo)
isFoldedOrDeadInstruction - Return true if the specified instruction is side-effect free and is eithe...
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file describes how to lower LLVM code to machine code.
DEMANGLE_DUMP_METHOD void dump() const
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
Class for arbitrary precision integers.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
An immutable pass that tracks lazily created AssumptionCache objects.
LLVM Basic Block Representation.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
InstListType::const_iterator const_iterator
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
bool isEHPad() const
Return true if this basic block is an exception handling block.
const Instruction * getFirstMayFaultInst() const
Returns the first potential AsynchEH faulty instruction currently it checks for loads/stores (which m...
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Legacy analysis pass which computes BranchProbabilityInfo.
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
This is an important base class in LLVM.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for ISel fallback path.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
void setLastLocalValue(MachineInstr *I)
Update the position of the last instruction emitted for materializing constants for use in the curren...
void handleDbgInfo(const Instruction *II)
Target-independent lowering of non-instruction debug info associated with this instruction.
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We're checking to see if we can fold LI into FoldInst.
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
void finishBasicBlock()
Flush the local value map.
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
unsigned arg_size() const
arg_size - Return the number of funcletpad arguments.
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th funcletpad argument.
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
SmallPtrSet< const DbgVariableRecord *, 8 > PreprocessedDVRDeclares
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
SmallPtrSet< const DbgDeclareInst *, 8 > PreprocessedDbgDeclares
Collection of dbg.declare instructions handled after argument lowering and before ISel proper.
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineRegisterInfo * RegInfo
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Data structure describing the variable locations in a function.
const VarLocInfo * single_locs_begin() const
DILocalVariable * getDILocalVariable(const VarLocInfo *Loc) const
Return the DILocalVariable for the location definition represented by ID.
const VarLocInfo * single_locs_end() const
One past the last single-location variable location definition.
const BasicBlock & getEntryBlock() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
DISubprogram * getSubprogram() const
Get the attached subprogram.
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool callsFunctionThatReturnsTwice() const
callsFunctionThatReturnsTwice - Return true if the function has a call to setjmp or other function th...
An analysis pass which caches information about the entire Module.
Module * getParent()
Get the module that this global value is contained inside of...
This class is used to form a handle around another node that is persistent and is updated across invo...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
A wrapper class for inspecting calls to intrinsic functions.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
static void getLazyBFIAnalysisUsage(AnalysisUsage &AU)
Helper for client passes to set up the analysis usage on behalf of this pass.
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Describe properties that are true of each instruction in the target description file.
bool mayStore() const
Return true if this instruction could possibly modify memory.
bool mayLoad() const
Return true if this instruction could possibly read memory.
bool mayRaiseFPException() const
Return true if this instruction may raise a floating-point exception.
bool isCall() const
Return true if the instruction is a call.
bool isReturn() const
Return true if the instruction is a return.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
const MDNode * getMD() const
const MDOperand & getOperand(unsigned I) const
StringRef getString() const
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
instr_iterator instr_end()
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasCalls() const
Return true if the current function has any function calls.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
bool hasProperty(Property P) const
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setExposesReturnsTwice(bool B)
setCallsSetJmp - Set a flag that indicates if there's a call to a "returns twice" function.
void setHasInlineAsm(bool B)
Set a flag that indicates that the function contains inline assembly.
void setWasmLandingPadIndex(const MachineBasicBlock *LPad, unsigned Index)
Map the landing pad to its index. Used for Wasm exception handling.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
bool hasInlineAsm() const
Returns true if the function contains any inline assembly.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void finalizeDebugInstrRefs()
Finalise any partially emitted debug instructions.
void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef< unsigned > Sites)
Map the landing pad's EH symbol to the call site indexes.
void setUseDebugInstrRef(bool UseInstrRef)
Set whether this function will use instruction referencing or not.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
bool shouldUseDebugInstrRef() const
Determine whether, in the current machine configuration, we should use instruction referencing or not...
const MachineFunctionProperties & getProperties() const
Get the function properties.
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable in a stack slot.
const MachineBasicBlock & front() const
void print(raw_ostream &OS, const SlotIndexes *=nullptr) const
print - Print out the MachineFunction in a format suitable for debugging to the specified stream.
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
const MachineBasicBlock * getParent() const
bool isDebugValue() const
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
This class contains meta information specific to a module.
const MCContext & getContext() const
bool usesMSVCFloatingPoint() const
void setUsesMSVCFloatingPoint(bool b)
Register getReg() const
getReg - Returns the register number.
MachinePassRegistry - Track the registration of machine passes.
defusechain_iterator - This class provides iterator support for machine operands in the function that...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
void EmitLiveInCopies(MachineBasicBlock *EntryMBB, const TargetRegisterInfo &TRI, const TargetInstrInfo &TII)
EmitLiveInCopies - Emit copies to initialize livein virtual registers into the given entry block.
use_instr_iterator use_instr_begin(Register RegNo) const
ArrayRef< std::pair< MCRegister, Register > > liveins() const
static use_instr_iterator use_instr_end()
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
This class is used by SelectionDAGISel to temporarily override the optimization level on a per-functi...
OptLevelChanger(SelectionDAGISel &ISel, CodeGenOptLevel NewOptLevel)
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
RegisterPassParser class - Handle the addition of new machine passes.
ScheduleDAGSDNodes *(*)(SelectionDAGISel *, CodeGenOptLevel) FunctionPassCtor
static MachinePassRegistry< FunctionPassCtor > Registry
RegisterScheduler class - Track the registration of instruction schedulers.
Wrapper class representing virtual and physical registers.
static unsigned virtReg2Index(Register Reg)
Convert a virtual register number to a 0-based index.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
void setNodeId(int Id)
Set unique node id.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckOrMask - The isel is trying to match something like (or X, 255).
const TargetLowering * TLI
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
virtual bool CheckNodePredicate(SDNode *N, unsigned PredNo) const
CheckNodePredicate - This function is generated by tblgen in the target.
std::unique_ptr< OptimizationRemarkEmitter > ORE
Current optimization remark emitter.
MachineRegisterInfo * RegInfo
unsigned DAGSize
DAGSize - Size of DAG being instruction selected.
@ OPC_MorphNodeTo2GlueOutput
@ OPC_CheckPatternPredicate5
@ OPC_EmitCopyToRegTwoByte
@ OPC_MorphNodeTo2GlueInput
@ OPC_CheckChild2CondCode
@ OPC_CheckPatternPredicateTwoByte
@ OPC_CheckPatternPredicate1
@ OPC_MorphNodeTo1GlueOutput
@ OPC_EmitMergeInputChains1_1
@ OPC_CheckPatternPredicate2
@ OPC_EmitConvertToTarget2
@ OPC_EmitConvertToTarget0
@ OPC_CheckPatternPredicate4
@ OPC_EmitConvertToTarget1
@ OPC_CheckPatternPredicate
@ OPC_MorphNodeTo0GlueInput
@ OPC_CheckPatternPredicate6
@ OPC_MorphNodeTo0GlueOutput
@ OPC_CheckPatternPredicate7
@ OPC_EmitMergeInputChains
@ OPC_EmitMergeInputChains1_0
@ OPC_CheckFoldableChainNode
@ OPC_EmitConvertToTarget3
@ OPC_CheckPredicateWithOperands
@ OPC_EmitConvertToTarget4
@ OPC_EmitStringInteger32
@ OPC_EmitConvertToTarget7
@ OPC_EmitMergeInputChains1_2
@ OPC_EmitConvertToTarget5
@ OPC_CheckPatternPredicate0
@ OPC_MorphNodeTo1GlueInput
@ OPC_CheckPatternPredicate3
@ OPC_EmitConvertToTarget
@ OPC_EmitConvertToTarget6
bool isOrEquivalentToAdd(const SDNode *N) const
virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N, unsigned PatternNo, SmallVectorImpl< std::pair< SDValue, SDNode * > > &Result)
virtual bool CheckPatternPredicate(unsigned PredNo) const
CheckPatternPredicate - This function is generated by tblgen in the target.
static int getNumFixedFromVariadicInfo(unsigned Flags)
getNumFixedFromVariadicInfo - Transform an EmitNode flags word into the number of fixed arity values ...
const TargetLibraryInfo * LibInfo
static int getUninvalidatedNodeId(SDNode *N)
const TargetInstrInfo * TII
virtual bool CheckNodePredicateWithOperands(SDNode *N, unsigned PredNo, const SmallVectorImpl< SDValue > &Operands) const
CheckNodePredicateWithOperands - This function is generated by tblgen in the target.
void SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, unsigned TableSize)
static void EnforceNodeIdInvariant(SDNode *N)
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo)
bool MatchFilterFuncName
True if the function currently processing is in the function printing list (i.e.
void SelectInlineAsmMemoryOperands(std::vector< SDValue > &Ops, const SDLoc &DL)
SelectInlineAsmMemoryOperands - Calls to this are automatically generated by tblgen.
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOptLevel OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool CheckAndMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckAndMask - The isel is trying to match something like (and X, 255).
SwiftErrorValueTracking * SwiftError
SelectionDAGISel(char &ID, TargetMachine &tm, CodeGenOptLevel OL=CodeGenOptLevel::Default)
virtual StringRef getPatternForIndex(unsigned index)
getPatternForIndex - Patterns selected by tablegen during ISEL
bool mayRaiseFPException(SDNode *Node) const
Return whether the node may raise an FP exception.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
std::unique_ptr< SelectionDAGBuilder > SDB
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
static void InvalidateNodeId(SDNode *N)
~SelectionDAGISel() override
virtual StringRef getIncludePathForIndex(unsigned index)
getIncludePathForIndex - get the td source location of pattern instantiation
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
void setFunctionLoweringInfo(FunctionLoweringInfo *FuncInfo)
SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
void VerifyDAGDivergence()
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
void salvageDebugInfo(SDNode &N)
To be invoked on an SDNode that is slated to be erased.
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
allnodes_const_iterator allnodes_begin() const
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
void viewGraph(const std::string &Title)
Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
void Combine(CombineLevel Level, AAResults *AA, CodeGenOptLevel OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together,...
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
SDValue getRegister(unsigned Reg, EVT VT)
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, UniformityInfo *UA, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs)
Prepare this SelectionDAG to process code in the given MachineFunction.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
const FunctionVarLocs * getFunctionVarLocs() const
Returns the result of the AssignmentTrackingAnalysis pass if it's available, otherwise return nullptr...
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
ilist< SDNode >::iterator allnodes_iterator
bool LegalizeTypes()
This transforms the SelectionDAG into a SelectionDAG that only uses types natively supported by the t...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
bool shouldEmitSDCheck(const BasicBlock &BB) const
void copyToMachineFrameInfo(MachineFrameInfo &MFI) const
StringRef - Represent a constant reference to a string, i.e.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
bool createEntriesInEntryBlock(DebugLoc DbgLoc)
Create initial definitions of swifterror values in the entry block of the current function.
void setFunction(MachineFunction &MF)
Initialize data structures for specified new function.
void preassignVRegs(MachineBasicBlock *MBB, BasicBlock::const_iterator Begin, BasicBlock::const_iterator End)
void propagateVRegs()
Propagate assigned swifterror vregs through a function, synthesizing PHI nodes when needed to maintai...
TargetIntrinsicInfo - Interface to description of machine instruction set.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
bool isStrictFPEnabled() const
Return true if the target support strict float operation.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
virtual void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const
Insert explicit copies in entry and exit blocks.
virtual void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
virtual void initializeSplitCSR(MachineBasicBlock *Entry) const
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
virtual FastISel * createFastISel(FunctionLoweringInfo &, const TargetLibraryInfo *) const
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
virtual bool supportSplitCSR(MachineFunction *MF) const
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
Primary interface to the complete machine description for the target machine.
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
const Triple & getTargetTriple() const
void setFastISel(bool Enable)
bool getO0WantsFastISel()
void resetTargetOptions(const Function &F) const
Reset the target options based on the function's attributes.
void setOptLevel(CodeGenOptLevel Level)
Overrides the optimization level.
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetLowering * getTargetLowering() const
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
LLVM Value Representation.
bool hasOneUse() const
Return true if there is exactly one use of this value.
iterator_range< user_iterator > users()
StringRef getName() const
Return a constant reference to the value's name.
self_iterator getIterator()
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const CustomOperand< const MCSubtargetInfo & > Msg[]
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ JUMP_TABLE_DEBUG_INFO
JUMP_TABLE_DEBUG_INFO - Jumptable debug info.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
@ UNDEF
UNDEF - An undefined node.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ LIFETIME_START
This corresponds to the llvm.lifetime.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
ScheduleDAGSDNodes * createDefaultScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createDefaultScheduler - This creates an instruction scheduler appropriate for the target.
bool succ_empty(const Instruction *I)
ScheduleDAGSDNodes * createBURRListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createBURRListDAGScheduler - This creates a bottom up register usage reduction list scheduler.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
ScheduleDAGSDNodes * createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel)
createHybridListDAGScheduler - This creates a bottom up register pressure aware list scheduler that m...
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
MachineBasicBlock::iterator findSplitPointForStackProtector(MachineBasicBlock *BB, const TargetInstrInfo &TII)
Find the split point at which to splice the end of BB into its success stack protector check machine ...
bool TimePassesIsEnabled
If the user specifies the -time-passes argument on an LLVM tool command line then the value of this b...
LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry &)
ScheduleDAGSDNodes * createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createFastDAGScheduler - This creates a "fast" scheduler.
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
ScheduleDAGSDNodes * createDAGLinearizer(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createDAGLinearizer - This creates a "no-scheduling" scheduler which linearize the DAG using topologi...
void initializeAAResultsWrapperPassPass(PassRegistry &)
void initializeGCModuleInfoPass(PassRegistry &)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isFunctionInPrintList(StringRef FunctionName)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
CodeGenOptLevel
Code generation optimization level.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
ScheduleDAGSDNodes * createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createSourceListDAGScheduler - This creates a bottom up list scheduler that schedules nodes in source...
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
ScheduleDAGSDNodes * createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel)
createILPListDAGScheduler - This creates a bottom up register pressure aware list scheduler that trie...
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &)
ScheduleDAGSDNodes * createVLIWDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createVLIWDAGScheduler - Scheduler for VLIW targets.
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class is basically a combination of TimeRegion and Timer.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)
DenseMap< const BasicBlock *, int > BlockToStateMap