78#include "llvm/IR/IntrinsicsWebAssembly.h"
115#define DEBUG_TYPE "isel"
117STATISTIC(NumFastIselFailures,
"Number of instructions fast isel failed on");
118STATISTIC(NumFastIselSuccess,
"Number of instructions fast isel selected");
119STATISTIC(NumFastIselBlocks,
"Number of blocks selected entirely by fast isel");
120STATISTIC(NumDAGBlocks,
"Number of blocks selected using DAG");
121STATISTIC(NumDAGIselRetries,
"Number of times dag isel has to try another path");
122STATISTIC(NumEntryBlocks,
"Number of entry blocks encountered");
124 "Number of entry blocks where fast isel failed to lower arguments");
128 cl::desc(
"Enable abort calls when \"fast\" instruction selection "
129 "fails to lower an instruction: 0 disable the abort, 1 will "
130 "abort but for args, calls and terminators, 2 will also "
131 "abort for argument lowering, and 3 will never fallback "
132 "to SelectionDAG."));
136 cl::desc(
"Emit a diagnostic when \"fast\" instruction selection "
137 "falls back to SelectionDAG."));
141 cl::desc(
"use Machine Branch Probability Info"),
147 cl::desc(
"Only display the basic block whose name "
148 "matches this for all view-*-dags options"));
151 cl::desc(
"Pop up a window to show dags before the first "
152 "dag combine pass"));
155 cl::desc(
"Pop up a window to show dags before legalize types"));
158 cl::desc(
"Pop up a window to show dags before the post "
159 "legalize types dag combine pass"));
162 cl::desc(
"Pop up a window to show dags before legalize"));
165 cl::desc(
"Pop up a window to show dags before the second "
166 "dag combine pass"));
169 cl::desc(
"Pop up a window to show isel dags as they are selected"));
172 cl::desc(
"Pop up a window to show sched dags as they are processed"));
175 cl::desc(
"Pop up a window to show SUnit dags after they are processed"));
200 cl::desc(
"Instruction schedulers available (before register"
222 if (NewOptLevel == SavedOptLevel)
226 LLVM_DEBUG(
dbgs() <<
"\nChanging optimization level for Function "
228 LLVM_DEBUG(
dbgs() <<
"\tBefore: -O" << SavedOptLevel <<
" ; After: -O"
229 << NewOptLevel <<
"\n");
233 dbgs() <<
"\tFastISel is "
242 LLVM_DEBUG(
dbgs() <<
"\nRestoring optimization level for Function "
245 << SavedOptLevel <<
"\n");
261 if (
auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
262 return SchedulerCtor(IS, OptLevel);
266 (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||
280 "Unknown sched type!");
299 dbgs() <<
"If a target marks an instruction with "
300 "'usesCustomInserter', it must implement "
301 "TargetLowering::EmitInstrWithCustomInserter!\n";
309 "If a target marks an instruction with 'hasPostISelHook', "
310 "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
361 if (!TT.isWindowsMSVCEnvironment())
369 if (
I.getType()->isFPOrFPVectorTy()) {
373 for (
const auto &Op :
I.operands()) {
374 if (Op->getType()->isFPOrFPVectorTy()) {
389 "-fast-isel-abort > 0 requires -fast-isel");
414 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(Fn);
415 GFI = Fn.
hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) :
nullptr;
416 ORE = std::make_unique<OptimizationRemarkEmitter>(&Fn);
417 AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(mf.
getFunction());
418 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
421 BFI = &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI();
425 FnVarLocs = getAnalysis<AssignmentTrackingAnalysis>().getResults();
430 if (
auto *UAPass = getAnalysisIfAvailable<UniformityInfoWrapperPass>())
431 UA = &UAPass->getUniformityInfo();
442 FuncInfo->BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
447 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
468 if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))
482 SelectAllBasicBlocks(Fn);
510 MRI.constrainRegClass(To,
MRI.getRegClass(
From));
516 if (!
MRI.use_empty(To))
536 if (Term !=
MBB.
end() && Term->isReturn()) {
545 if (!
FuncInfo->ArgDbgValues.empty())
551 for (
unsigned i = 0, e =
FuncInfo->ArgDbgValues.size(); i != e; ++i) {
553 assert(
MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
554 "Function parameters should not be described by DBG_VALUE_LIST.");
555 bool hasFI =
MI->getDebugOperand(0).isFI();
557 hasFI ?
TRI.getFrameRegister(*
MF) :
MI->getDebugOperand(0).getReg();
558 if (Reg.isPhysical())
565 Def->getParent()->insert(std::next(InsertPos),
MI);
577 if (LDI != LiveInMap.
end()) {
578 assert(!hasFI &&
"There's no handling of frame pointer updating here yet "
582 const MDNode *Variable =
MI->getDebugVariable();
583 const MDNode *Expr =
MI->getDebugExpression();
585 bool IsIndirect =
MI->isIndirectDebugValue();
587 assert(
MI->getDebugOffset().getImm() == 0 &&
588 "DBG_VALUE with nonzero offset");
589 assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(
DL) &&
590 "Expected inlined-at fields to agree");
591 assert(
MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
592 "Didn't expect to see a DBG_VALUE_LIST here");
595 IsIndirect, LDI->second, Variable, Expr);
607 CopyUseMI =
UseMI;
continue;
610 CopyUseMI =
nullptr;
break;
613 TRI.getRegSizeInBits(LDI->second,
MRI) ==
633 for (
const auto &
MBB : *
MF) {
637 for (
const auto &
MI :
MBB) {
640 MI.isStackAligningInlineAsm()) {
643 if (
MI.isInlineAsm()) {
671 if (!R.getLocation().isValid() || ShouldAbort)
672 R << (
" (in function: " + MF.
getName() +
")").str();
696 HadTailCall =
SDB->HasTailCall;
697 SDB->resolveOrClearDbgInfo();
704void SelectionDAGISel::ComputeLiveOutVRegInfo() {
717 for (
const SDValue &Op :
N->op_values())
725 unsigned DestReg = cast<RegisterSDNode>(
N->getOperand(1))->getReg();
731 EVT SrcVT = Src.getValueType();
737 FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);
738 }
while (!Worklist.
empty());
741void SelectionDAGISel::CodeGenAndEmitDAG() {
743 StringRef GroupDescription =
"Instruction Selection and Scheduling";
744 std::string BlockName;
745 bool MatchFilterBB =
false; (void)MatchFilterBB;
748 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*
FuncInfo->Fn);
757 FuncInfo->MBB->getBasicBlock()->getName());
939 ComputeLiveOutVRegInfo();
949 DoInstructionSelection();
985 if (FirstMBB != LastMBB)
986 SDB->UpdateSplitBlock(FirstMBB, LastMBB);
1008 :
SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
1021 void NodeInserted(
SDNode *
N)
override {
1022 SDNode *CurNode = &*ISelPosition;
1023 if (
MDNode *MD = DAG.getPCSections(CurNode))
1024 DAG.addPCSections(
N, MD);
1054 while (!Nodes.
empty()) {
1056 for (
auto *U :
N->uses()) {
1057 auto UId = U->getNodeId();
1070 int InvalidId = -(
N->getNodeId() + 1);
1071 N->setNodeId(InvalidId);
1076 int Id =
N->getNodeId();
1082void SelectionDAGISel::DoInstructionSelection() {
1085 <<
FuncInfo->MBB->getName() <<
"'\n");
1103 ISelUpdater ISU(*
CurDAG, ISelPosition);
1110 SDNode *Node = &*--ISelPosition;
1114 if (Node->use_empty())
1121 while (!Nodes.
empty()) {
1125 for (
const SDValue &Op :
N->op_values()) {
1137 assert(Op->getNodeId() != -1 &&
1138 "Node has already selected predecessor node");
1155 switch (
Node->getOpcode()) {
1164 ActionVT =
Node->getOperand(1).getValueType();
1167 ActionVT =
Node->getValueType(0);
1175 LLVM_DEBUG(
dbgs() <<
"\nISEL: Starting selection on root node: ";
1191 if (
const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {
1193 if (IID == Intrinsic::eh_exceptionpointer ||
1194 IID == Intrinsic::eh_exceptioncode)
1209 bool IsSingleCatchAllClause =
1214 bool IsCatchLongjmp = CPI->
arg_size() == 0;
1215 if (!IsSingleCatchAllClause && !IsCatchLongjmp) {
1217 bool IntrFound =
false;
1219 if (
const auto *Call = dyn_cast<IntrinsicInst>(U)) {
1221 if (IID == Intrinsic::wasm_landingpad_index) {
1222 Value *IndexArg = Call->getArgOperand(1);
1223 int Index = cast<ConstantInt>(IndexArg)->getZExtValue();
1230 assert(IntrFound &&
"wasm.landingpad.index intrinsic not found!");
1237bool SelectionDAGISel::PrepareEHLandingPad() {
1249 if (
const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->
getFirstNonPHI())) {
1254 assert(EHPhysReg &&
"target lacks exception pointer register");
1256 unsigned VReg =
FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
1258 TII->
get(TargetOpcode::COPY), VReg)
1276 if (
auto *RegMask =
TRI.getCustomEHPadPreservedMask(*
MF))
1280 if (
const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->
getFirstNonPHI()))
1318 TII->
get(TargetOpcode::EH_LABEL))
1327 TII->
get(TargetOpcode::EH_LABEL))
1338 return !
I->mayWriteToMemory() &&
1339 !
I->isTerminator() &&
1340 !isa<DbgInfoIntrinsic>(
I) &&
1351 assert(Var &&
"Missing variable");
1352 assert(DbgLoc &&
"Missing location");
1362 int FI = std::numeric_limits<int>::max();
1363 if (
const auto *AI = dyn_cast<AllocaInst>(
Address)) {
1367 }
else if (
const auto *
Arg = dyn_cast<Argument>(
Address))
1370 if (FI == std::numeric_limits<int>::max())
1373 if (
Offset.getBoolValue())
1377 LLVM_DEBUG(
dbgs() <<
"processDbgDeclare: setVariableDbgInfo Var=" << *Var
1378 <<
", Expr=" << *Expr <<
", FI=" << FI
1379 <<
", DbgLoc=" << DbgLoc <<
"\n");
1392 <<
" (bad address)\n");
1396 DI->getVariable(), DI->getDebugLoc());
1410 assert(!It->Values.hasArgList() &&
"Single loc variadic ops not supported");
1416void SelectionDAGISel::SelectAllBasicBlocks(
const Function &Fn) {
1446 ++NumFastIselFailLowerArguments;
1451 R <<
"FastISel didn't lower all arguments: "
1459 CodeGenAndEmitDAG();
1473 if (FastIS && Inserted)
1478 "expected AssignmentTrackingAnalysis pass results");
1488 bool AllPredsVisited =
true;
1490 if (!
FuncInfo->VisitedBBs.count(Pred)) {
1491 AllPredsVisited =
false;
1496 if (AllPredsVisited) {
1498 FuncInfo->ComputePHILiveOutRegInfo(&PN);
1501 FuncInfo->InvalidatePHILiveOutRegInfo(&PN);
1504 FuncInfo->VisitedBBs.insert(LLVMBB);
1520 FuncInfo->ExceptionPointerVirtReg = 0;
1521 FuncInfo->ExceptionSelectorVirtReg = 0;
1523 if (!PrepareEHLandingPad())
1531 unsigned NumFastIselRemaining = std::distance(Begin, End);
1537 for (; BI != Begin; --BI) {
1543 --NumFastIselRemaining;
1553 --NumFastIselRemaining;
1554 ++NumFastIselSuccess;
1559 while (BeforeInst != &*Begin) {
1564 if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1569 <<
"FastISel folded load: " << *BeforeInst <<
"\n");
1571 --NumFastIselRemaining;
1572 ++NumFastIselSuccess;
1584 if (isa<CallInst>(Inst) && !isa<GCStatepointInst>(Inst) &&
1585 !isa<GCRelocateInst>(Inst) && !isa<GCResultInst>(Inst)) {
1589 R <<
"FastISel missed call";
1592 std::string InstStrStorage;
1596 R <<
": " << InstStr.str();
1601 if (!Inst->getType()->isVoidTy() && !Inst->getType()->isTokenTy() &&
1602 !Inst->use_empty()) {
1608 bool HadTailCall =
false;
1610 SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);
1622 unsigned RemainingNow = std::distance(Begin, BI);
1623 NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1624 NumFastIselRemaining = RemainingNow;
1629 Inst->getDebugLoc(), LLVMBB);
1632 if (Inst->isTerminator()) {
1634 R <<
"FastISel missed terminator";
1638 R <<
"FastISel missed";
1642 std::string InstStrStorage;
1645 R <<
": " << InstStr.str();
1650 NumFastIselFailures += NumFastIselRemaining;
1658 bool FunctionBasedInstrumentation =
1660 SDB->SPDescriptor.initialize(LLVMBB,
FuncInfo->MBBMap[LLVMBB],
1661 FunctionBasedInstrumentation);
1667 ++NumFastIselBlocks;
1674 SelectBasicBlock(Begin, BI, HadTailCall);
1686 FuncInfo->PHINodesToUpdate.clear();
1692 reportIPToStateForBlocks(
MF);
1699 SDB->clearDanglingDebugInfo();
1700 SDB->SPDescriptor.resetPerFunctionState();
1704SelectionDAGISel::FinishBasicBlock() {
1706 <<
FuncInfo->PHINodesToUpdate.size() <<
"\n";
1707 for (
unsigned i = 0, e =
FuncInfo->PHINodesToUpdate.size(); i != e;
1709 <<
"Node " << i <<
" : (" <<
FuncInfo->PHINodesToUpdate[i].first
1710 <<
", " <<
FuncInfo->PHINodesToUpdate[i].second <<
")\n");
1714 for (
unsigned i = 0, e =
FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1717 "This is not a machine PHI node that we are updating!");
1718 if (!
FuncInfo->MBB->isSuccessor(
PHI->getParent()))
1724 if (
SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
1733 SDB->visitSPDescriptorParent(
SDB->SPDescriptor, ParentMBB);
1736 CodeGenAndEmitDAG();
1739 SDB->SPDescriptor.resetPerBBState();
1740 }
else if (
SDB->SPDescriptor.shouldEmitStackProtector()) {
1754 SuccessMBB->
splice(SuccessMBB->
end(), ParentMBB,
1761 SDB->visitSPDescriptorParent(
SDB->SPDescriptor, ParentMBB);
1764 CodeGenAndEmitDAG();
1768 if (FailureMBB->
empty()) {
1771 SDB->visitSPDescriptorFailure(
SDB->SPDescriptor);
1774 CodeGenAndEmitDAG();
1778 SDB->SPDescriptor.resetPerBBState();
1782 for (
auto &BTB :
SDB->SL->BitTestCases) {
1792 CodeGenAndEmitDAG();
1796 for (
unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
1797 UnhandledProb -= BTB.Cases[
j].ExtraProb;
1812 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1815 NextMBB = BTB.Cases[
j + 1].TargetBB;
1816 }
else if (j + 1 == ej) {
1818 NextMBB = BTB.Default;
1821 NextMBB = BTB.Cases[
j + 1].ThisBB;
1824 SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
1829 CodeGenAndEmitDAG();
1831 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
1833 BTB.Cases.pop_back();
1839 for (
const std::pair<MachineInstr *, unsigned> &
P :
1844 "This is not a machine PHI node that we are updating!");
1847 if (PHIBB == BTB.Default) {
1848 PHI.addReg(
P.second).addMBB(BTB.Parent);
1849 if (!BTB.ContiguousRange) {
1850 PHI.addReg(
P.second).addMBB(BTB.Cases.back().ThisBB);
1857 PHI.addReg(
P.second).addMBB(cBB);
1861 SDB->SL->BitTestCases.clear();
1866 for (
unsigned i = 0, e =
SDB->SL->JTCases.size(); i != e; ++i) {
1868 if (!
SDB->SL->JTCases[i].first.Emitted) {
1870 FuncInfo->MBB =
SDB->SL->JTCases[i].first.HeaderBB;
1873 SDB->visitJumpTableHeader(
SDB->SL->JTCases[i].second,
1877 CodeGenAndEmitDAG();
1884 SDB->visitJumpTable(
SDB->SL->JTCases[i].second);
1887 CodeGenAndEmitDAG();
1890 for (
unsigned pi = 0, pe =
FuncInfo->PHINodesToUpdate.size();
1895 "This is not a machine PHI node that we are updating!");
1897 if (PHIBB ==
SDB->SL->JTCases[i].second.Default)
1899 .addMBB(
SDB->SL->JTCases[i].first.HeaderBB);
1901 if (
FuncInfo->MBB->isSuccessor(PHIBB))
1905 SDB->SL->JTCases.clear();
1909 for (
unsigned i = 0, e =
SDB->SL->SwitchCases.size(); i != e; ++i) {
1917 if (
SDB->SL->SwitchCases[i].TrueBB !=
SDB->SL->SwitchCases[i].FalseBB)
1924 CodeGenAndEmitDAG();
1934 for (
unsigned i = 0, e = Succs.
size(); i != e; ++i) {
1945 for (
unsigned pn = 0; ; ++pn) {
1947 "Didn't find PHI entry!");
1948 if (
FuncInfo->PHINodesToUpdate[pn].first ==
PHI) {
1949 PHI.addReg(
FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
1957 SDB->SL->SwitchCases.clear();
1978 int64_t DesiredMaskS)
const {
1979 const APInt &ActualMask =
RHS->getAPIntValue();
1980 const APInt &DesiredMask =
APInt(
LHS.getValueSizeInBits(), DesiredMaskS);
1983 if (ActualMask == DesiredMask)
1992 APInt NeededMask = DesiredMask & ~ActualMask;
2007 int64_t DesiredMaskS)
const {
2008 const APInt &ActualMask =
RHS->getAPIntValue();
2009 const APInt &DesiredMask =
APInt(
LHS.getValueSizeInBits(), DesiredMaskS);
2012 if (ActualMask == DesiredMask)
2021 APInt NeededMask = DesiredMask & ~ActualMask;
2038 std::vector<SDValue> InOps;
2047 if (InOps[e-1].getValueType() ==
MVT::Glue)
2051 unsigned Flags = cast<ConstantSDNode>(InOps[i])->getZExtValue();
2054 Ops.insert(Ops.end(), InOps.begin()+i,
2059 "Memory operand with multiple values?");
2061 unsigned TiedToOperand;
2065 Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2066 for (; TiedToOperand; --TiedToOperand) {
2068 Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2073 std::vector<SDValue> SelOps;
2092 if (e != InOps.size())
2093 Ops.push_back(InOps.back());
2100 unsigned FlagResNo =
N->getNumValues()-1;
2103 if (
Use.getResNo() == FlagResNo)
2112 bool IgnoreChains) {
2121 Visited.
insert(ImmedUse);
2126 if ((Op.getValueType() ==
MVT::Other && IgnoreChains) ||
N == Def)
2128 if (!Visited.
insert(
N).second)
2134 if (Root != ImmedUse) {
2138 if ((Op.getValueType() ==
MVT::Other && IgnoreChains) ||
N == Def)
2140 if (!Visited.
insert(
N).second)
2154 return N.hasOneUse();
2161 bool IgnoreChains) {
2220 IgnoreChains =
false;
2226void SelectionDAGISel::Select_INLINEASM(
SDNode *
N) {
2229 std::vector<SDValue> Ops(
N->op_begin(),
N->op_end());
2239void SelectionDAGISel::Select_READ_REGISTER(
SDNode *Op) {
2244 EVT VT =
Op->getValueType(0);
2250 Op->getOperand(0), dl, Reg,
Op->getValueType(0));
2256void SelectionDAGISel::Select_WRITE_REGISTER(
SDNode *Op) {
2261 EVT VT =
Op->getOperand(2).getValueType();
2267 Op->getOperand(0), dl, Reg,
Op->getOperand(2));
2273void SelectionDAGISel::Select_UNDEF(
SDNode *
N) {
2277void SelectionDAGISel::Select_FREEZE(
SDNode *
N) {
2285void SelectionDAGISel::Select_ARITH_FENCE(
SDNode *
N) {
2290void SelectionDAGISel::Select_MEMBARRIER(
SDNode *
N) {
2314void SelectionDAGISel::Select_STACKMAP(
SDNode *
N) {
2316 auto *It =
N->op_begin();
2334 for (; It !=
N->op_end(); It++)
2335 pushStackMapLiveVariable(Ops, *It,
DL);
2344void SelectionDAGISel::Select_PATCHPOINT(
SDNode *
N) {
2346 auto *It =
N->op_begin();
2351 std::optional<SDValue> Glue;
2378 for (
uint64_t I = cast<ConstantSDNode>(NumArgs)->getZExtValue();
I != 0;
I--)
2382 for (; It !=
N->op_end(); It++)
2383 pushStackMapLiveVariable(Ops, *It,
DL);
2388 if (Glue.has_value())
2398 assert(Val >= 128 &&
"Not a VBR");
2404 NextBits = MatcherTable[
Idx++];
2405 Val |= (NextBits&127) << Shift;
2407 }
while (NextBits & 128);
2414void SelectionDAGISel::UpdateChains(
2421 if (!ChainNodesMatched.
empty()) {
2423 "Matched input chains but didn't produce a chain");
2426 for (
unsigned i = 0, e = ChainNodesMatched.
size(); i != e; ++i) {
2427 SDNode *ChainNode = ChainNodesMatched[i];
2434 "Deleted node left in chain");
2438 if (ChainNode == NodeToMatch && isMorphNodeTo)
2447 std::replace(ChainNodesMatched.
begin(), ChainNodesMatched.
end(),
N,
2448 static_cast<SDNode *
>(
nullptr));
2454 if (ChainNode != NodeToMatch && ChainNode->
use_empty() &&
2460 if (!NowDeadNodes.
empty())
2479 unsigned int Max = 8192;
2482 if (ChainNodesMatched.
size() == 1)
2483 return ChainNodesMatched[0]->getOperand(0);
2487 std::function<void(
const SDValue)> AddChains = [&](
const SDValue V) {
2492 if (!Visited.
insert(V.getNode()).second)
2495 for (
const SDValue &Op : V->op_values())
2501 for (
auto *
N : ChainNodesMatched) {
2506 while (!Worklist.
empty())
2510 if (InputChains.
size() == 0)
2520 for (
auto *
N : ChainNodesMatched)
2525 if (InputChains.
size() == 1)
2526 return InputChains[0];
2532SDNode *SelectionDAGISel::
2541 int OldGlueResultNo = -1, OldChainResultNo = -1;
2543 unsigned NTMNumResults =
Node->getNumValues();
2545 OldGlueResultNo = NTMNumResults-1;
2546 if (NTMNumResults != 1 &&
2548 OldChainResultNo = NTMNumResults-2;
2550 OldChainResultNo = NTMNumResults-1;
2568 (
unsigned)OldGlueResultNo != ResNumResults-1)
2570 SDValue(Res, ResNumResults - 1));
2576 if ((EmitNodeInfo &
OPFL_Chain) && OldChainResultNo != -1 &&
2577 (
unsigned)OldChainResultNo != ResNumResults-1)
2579 SDValue(Res, ResNumResults - 1));
2597 unsigned RecNo = MatcherTable[MatcherIndex++];
2598 assert(RecNo < RecordedNodes.size() &&
"Invalid CheckSame");
2599 return N == RecordedNodes[RecNo].first;
2604 const unsigned char *MatcherTable,
unsigned &MatcherIndex,
SDValue N,
2607 if (ChildNo >=
N.getNumOperands())
2609 return ::CheckSame(MatcherTable, MatcherIndex,
N.getOperand(ChildNo),
2630 uint16_t Opc = MatcherTable[MatcherIndex++];
2631 Opc |= (
unsigned short)MatcherTable[MatcherIndex++] << 8;
2632 return N->getOpcode() == Opc;
2639 if (
N.getValueType() == VT)
return true;
2649 if (ChildNo >=
N.getNumOperands())
2651 return ::CheckType(MatcherTable, MatcherIndex,
N.getOperand(ChildNo), TLI,
2658 return cast<CondCodeSDNode>(
N)->get() ==
2665 if (2 >=
N.getNumOperands())
2667 return ::CheckCondCode(MatcherTable, MatcherIndex,
N.getOperand(2));
2674 if (cast<VTSDNode>(
N)->getVT() == VT)
2695 int64_t Val = MatcherTable[MatcherIndex++];
2697 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2702 return C &&
C->getSExtValue() == Val;
2708 if (ChildNo >=
N.getNumOperands())
2710 return ::CheckInteger(MatcherTable, MatcherIndex,
N.getOperand(ChildNo));
2716 int64_t Val = MatcherTable[MatcherIndex++];
2718 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2720 if (
N->getOpcode() !=
ISD::AND)
return false;
2729 int64_t Val = MatcherTable[MatcherIndex++];
2731 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2733 if (
N->getOpcode() !=
ISD::OR)
return false;
2750 switch (Table[
Index++]) {
2778 unsigned Res = Table[
Index++];
2835 unsigned NumRecordedNodes;
2838 unsigned NumMatchedMemRefs;
2841 SDValue InputChain, InputGlue;
2844 bool HasChainNodesMatched;
2861 :
SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
2862 RecordedNodes(
RN), MatchScopes(MS) {}
2870 if (!
E ||
E->isMachineOpcode())
2873 if (
N == *NodeToMatch)
2878 for (
auto &
I : RecordedNodes)
2879 if (
I.first.getNode() ==
N)
2882 for (
auto &
I : MatchScopes)
2883 for (
auto &J :
I.NodeStack)
2884 if (J.getNode() ==
N)
2892 const unsigned char *MatcherTable,
2893 unsigned TableSize) {
2932 Select_INLINEASM(NodeToMatch);
2935 Select_READ_REGISTER(NodeToMatch);
2938 Select_WRITE_REGISTER(NodeToMatch);
2941 Select_UNDEF(NodeToMatch);
2944 Select_FREEZE(NodeToMatch);
2947 Select_ARITH_FENCE(NodeToMatch);
2950 Select_MEMBARRIER(NodeToMatch);
2953 Select_STACKMAP(NodeToMatch);
2956 Select_PATCHPOINT(NodeToMatch);
2983 SDValue InputChain, InputGlue;
2997 unsigned MatcherIndex = 0;
2999 if (!OpcodeOffset.empty()) {
3001 if (
N.getOpcode() < OpcodeOffset.size())
3002 MatcherIndex = OpcodeOffset[
N.getOpcode()];
3003 LLVM_DEBUG(
dbgs() <<
" Initial Opcode index to " << MatcherIndex <<
"\n");
3012 unsigned CaseSize = MatcherTable[
Idx++];
3014 CaseSize =
GetVBR(CaseSize, MatcherTable,
Idx);
3015 if (CaseSize == 0)
break;
3019 Opc |= (
unsigned short)MatcherTable[
Idx++] << 8;
3020 if (Opc >= OpcodeOffset.size())
3021 OpcodeOffset.resize((Opc+1)*2);
3022 OpcodeOffset[Opc] =
Idx;
3027 if (
N.getOpcode() < OpcodeOffset.size())
3028 MatcherIndex = OpcodeOffset[
N.getOpcode()];
3032 assert(MatcherIndex < TableSize &&
"Invalid index");
3034 unsigned CurrentOpcodeIndex = MatcherIndex;
3047 unsigned NumToSkip = MatcherTable[MatcherIndex++];
3048 if (NumToSkip & 128)
3049 NumToSkip =
GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3051 if (NumToSkip == 0) {
3056 FailIndex = MatcherIndex+NumToSkip;
3058 unsigned MatcherIndexOfPredicate = MatcherIndex;
3059 (void)MatcherIndexOfPredicate;
3066 Result, *
this, RecordedNodes);
3071 dbgs() <<
" Skipped scope entry (due to false predicate) at "
3072 <<
"index " << MatcherIndexOfPredicate <<
", continuing at "
3073 << FailIndex <<
"\n");
3074 ++NumDAGIselRetries;
3078 MatcherIndex = FailIndex;
3082 if (FailIndex == 0)
break;
3086 MatchScope NewEntry;
3087 NewEntry.FailIndex = FailIndex;
3088 NewEntry.NodeStack.append(NodeStack.
begin(), NodeStack.
end());
3089 NewEntry.NumRecordedNodes = RecordedNodes.
size();
3090 NewEntry.NumMatchedMemRefs = MatchedMemRefs.
size();
3091 NewEntry.InputChain = InputChain;
3092 NewEntry.InputGlue = InputGlue;
3093 NewEntry.HasChainNodesMatched = !ChainNodesMatched.
empty();
3099 SDNode *Parent =
nullptr;
3100 if (NodeStack.
size() > 1)
3101 Parent = NodeStack[NodeStack.
size()-2].getNode();
3102 RecordedNodes.
push_back(std::make_pair(
N, Parent));
3111 if (ChildNo >=
N.getNumOperands())
3114 RecordedNodes.
push_back(std::make_pair(
N->getOperand(ChildNo),
3119 if (
auto *MN = dyn_cast<MemSDNode>(
N))
3120 MatchedMemRefs.
push_back(MN->getMemOperand());
3130 if (
N->getNumOperands() != 0 &&
3131 N->getOperand(
N->getNumOperands()-1).getValueType() ==
MVT::Glue)
3132 InputGlue =
N->getOperand(
N->getNumOperands()-1);
3136 unsigned ChildNo = MatcherTable[MatcherIndex++];
3137 if (ChildNo >=
N.getNumOperands())
3139 N =
N.getOperand(ChildNo);
3149 if (ChildNo >=
N.getNumOperands())
3151 N =
N.getOperand(ChildNo);
3159 assert(!NodeStack.
empty() &&
"Node stack imbalance!");
3160 N = NodeStack.
back();
3164 if (!
::CheckSame(MatcherTable, MatcherIndex,
N, RecordedNodes))
break;
3183 unsigned OpNum = MatcherTable[MatcherIndex++];
3186 for (
unsigned i = 0; i < OpNum; ++i)
3187 Operands.push_back(RecordedNodes[MatcherTable[MatcherIndex++]].first);
3189 unsigned PredNo = MatcherTable[MatcherIndex++];
3195 unsigned CPNum = MatcherTable[MatcherIndex++];
3196 unsigned RecNo = MatcherTable[MatcherIndex++];
3197 assert(RecNo < RecordedNodes.
size() &&
"Invalid CheckComplexPat");
3201 std::unique_ptr<MatchStateUpdater> MSU;
3203 MSU.reset(
new MatchStateUpdater(*
CurDAG, &NodeToMatch, RecordedNodes,
3207 RecordedNodes[RecNo].first, CPNum,
3213 if (!
::CheckOpcode(MatcherTable, MatcherIndex,
N.getNode()))
break;
3223 unsigned Res = MatcherTable[MatcherIndex++];
3231 unsigned CurNodeOpcode =
N.getOpcode();
3232 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3236 CaseSize = MatcherTable[MatcherIndex++];
3238 CaseSize =
GetVBR(CaseSize, MatcherTable, MatcherIndex);
3239 if (CaseSize == 0)
break;
3241 uint16_t Opc = MatcherTable[MatcherIndex++];
3242 Opc |= (
unsigned short)MatcherTable[MatcherIndex++] << 8;
3245 if (CurNodeOpcode == Opc)
3249 MatcherIndex += CaseSize;
3253 if (CaseSize == 0)
break;
3256 LLVM_DEBUG(
dbgs() <<
" OpcodeSwitch from " << SwitchStart <<
" to "
3257 << MatcherIndex <<
"\n");
3262 MVT CurNodeVT =
N.getSimpleValueType();
3263 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3267 CaseSize = MatcherTable[MatcherIndex++];
3269 CaseSize =
GetVBR(CaseSize, MatcherTable, MatcherIndex);
3270 if (CaseSize == 0)
break;
3277 if (CurNodeVT == CaseVT)
3281 MatcherIndex += CaseSize;
3285 if (CaseSize == 0)
break;
3289 <<
"] from " << SwitchStart <<
" to " << MatcherIndex
3326 if (!
::CheckOrImm(MatcherTable, MatcherIndex,
N, *
this))
break;
3338 assert(NodeStack.
size() != 1 &&
"No parent node");
3341 bool HasMultipleUses =
false;
3342 for (
unsigned i = 1, e = NodeStack.
size()-1; i != e; ++i) {
3343 unsigned NNonChainUses = 0;
3344 SDNode *NS = NodeStack[i].getNode();
3346 if (UI.getUse().getValueType() !=
MVT::Other)
3347 if (++NNonChainUses > 1) {
3348 HasMultipleUses =
true;
3351 if (HasMultipleUses)
break;
3353 if (HasMultipleUses)
break;
3370 int64_t Val = MatcherTable[MatcherIndex++];
3372 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
3375 RecordedNodes.
push_back(std::pair<SDValue, SDNode*>(
3383 unsigned RegNo = MatcherTable[MatcherIndex++];
3384 RecordedNodes.
push_back(std::pair<SDValue, SDNode*>(
3394 unsigned RegNo = MatcherTable[MatcherIndex++];
3395 RegNo |= MatcherTable[MatcherIndex++] << 8;
3396 RecordedNodes.
push_back(std::pair<SDValue, SDNode*>(
3403 unsigned RecNo = MatcherTable[MatcherIndex++];
3404 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitConvertToTarget");
3405 SDValue Imm = RecordedNodes[RecNo].first;
3408 const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
3410 Imm.getValueType());
3412 const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
3414 Imm.getValueType());
3417 RecordedNodes.
push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3426 "EmitMergeInputChains should be the first chain producing node");
3428 "Should only have one EmitMergeInputChains per match");
3432 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitMergeInputChains");
3433 ChainNodesMatched.
push_back(RecordedNodes[RecNo].first.getNode());
3439 if (ChainNodesMatched.
back() != NodeToMatch &&
3440 !RecordedNodes[RecNo].first.hasOneUse()) {
3441 ChainNodesMatched.
clear();
3455 "EmitMergeInputChains should be the first chain producing node");
3462 unsigned NumChains = MatcherTable[MatcherIndex++];
3463 assert(NumChains != 0 &&
"Can't TF zero chains");
3466 "Should only have one EmitMergeInputChains per match");
3469 for (
unsigned i = 0; i != NumChains; ++i) {
3470 unsigned RecNo = MatcherTable[MatcherIndex++];
3471 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitMergeInputChains");
3472 ChainNodesMatched.
push_back(RecordedNodes[RecNo].first.getNode());
3478 if (ChainNodesMatched.
back() != NodeToMatch &&
3479 !RecordedNodes[RecNo].first.hasOneUse()) {
3480 ChainNodesMatched.
clear();
3486 if (ChainNodesMatched.
empty())
3500 unsigned RecNo = MatcherTable[MatcherIndex++];
3501 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitCopyToReg");
3502 unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3504 DestPhysReg |= MatcherTable[MatcherIndex++] << 8;
3510 DestPhysReg, RecordedNodes[RecNo].first,
3513 InputGlue = InputChain.
getValue(1);
3518 unsigned XFormNo = MatcherTable[MatcherIndex++];
3519 unsigned RecNo = MatcherTable[MatcherIndex++];
3520 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitNodeXForm");
3522 RecordedNodes.
push_back(std::pair<SDValue,SDNode*>(Res,
nullptr));
3528 unsigned index = MatcherTable[MatcherIndex++];
3529 index |= (MatcherTable[MatcherIndex++] << 8);
3538 uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3539 TargetOpc |= (
unsigned short)MatcherTable[MatcherIndex++] << 8;
3540 unsigned EmitNodeInfo = MatcherTable[MatcherIndex++];
3550 NumVTs = MatcherTable[MatcherIndex++];
3552 for (
unsigned i = 0; i != NumVTs; ++i) {
3568 if (VTs.
size() == 1)
3570 else if (VTs.
size() == 2)
3576 unsigned NumOps = MatcherTable[MatcherIndex++];
3578 for (
unsigned i = 0; i != NumOps; ++i) {
3579 unsigned RecNo = MatcherTable[MatcherIndex++];
3581 RecNo =
GetVBR(RecNo, MatcherTable, MatcherIndex);
3583 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitNode");
3584 Ops.
push_back(RecordedNodes[RecNo].first);
3591 FirstOpToCopy += (EmitNodeInfo &
OPFL_Chain) ? 1 : 0;
3593 "Invalid variadic node");
3596 for (
unsigned i = FirstOpToCopy, e = NodeToMatch->
getNumOperands();
3599 if (V.getValueType() ==
MVT::Glue)
break;
3614 bool MayRaiseFPException =
3623 if (!IsMorphNodeTo) {
3630 for (
unsigned i = 0, e = VTs.
size(); i != e; ++i) {
3637 "NodeToMatch was removed partway through selection");
3641 auto &Chain = ChainNodesMatched;
3643 "Chain node replaced during MorphNode");
3646 Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList,
3647 Ops, EmitNodeInfo));
3654 Flags.setNoFPExcept(
true);
3655 Res->setFlags(
Flags);
3677 bool mayLoad = MCID.
mayLoad();
3684 if (MMO->isLoad()) {
3687 }
else if (MMO->isStore()) {
3699 <<
" Dropping mem operands\n";
3700 dbgs() <<
" " << (IsMorphNodeTo ?
"Morphed" :
"Created")
3705 if (IsMorphNodeTo) {
3707 UpdateChains(Res, InputChain, ChainNodesMatched,
true);
3717 unsigned NumResults = MatcherTable[MatcherIndex++];
3719 for (
unsigned i = 0; i != NumResults; ++i) {
3720 unsigned ResSlot = MatcherTable[MatcherIndex++];
3722 ResSlot =
GetVBR(ResSlot, MatcherTable, MatcherIndex);
3724 assert(ResSlot < RecordedNodes.
size() &&
"Invalid CompleteMatch");
3725 SDValue Res = RecordedNodes[ResSlot].first;
3727 assert(i < NodeToMatch->getNumValues() &&
3730 "Invalid number of results to complete!");
3736 "invalid replacement");
3741 UpdateChains(NodeToMatch, InputChain, ChainNodesMatched,
false);
3754 "Didn't replace all uses of the node?");
3764 LLVM_DEBUG(
dbgs() <<
" Match failed at index " << CurrentOpcodeIndex
3766 ++NumDAGIselRetries;
3768 if (MatchScopes.
empty()) {
3769 CannotYetSelect(NodeToMatch);
3775 MatchScope &LastScope = MatchScopes.
back();
3776 RecordedNodes.
resize(LastScope.NumRecordedNodes);
3778 NodeStack.
append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
3779 N = NodeStack.
back();
3781 if (LastScope.NumMatchedMemRefs != MatchedMemRefs.
size())
3782 MatchedMemRefs.
resize(LastScope.NumMatchedMemRefs);
3783 MatcherIndex = LastScope.FailIndex;
3787 InputChain = LastScope.InputChain;
3788 InputGlue = LastScope.InputGlue;
3789 if (!LastScope.HasChainNodesMatched)
3790 ChainNodesMatched.
clear();
3795 unsigned NumToSkip = MatcherTable[MatcherIndex++];
3796 if (NumToSkip & 128)
3797 NumToSkip =
GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3801 if (NumToSkip != 0) {
3802 LastScope.FailIndex = MatcherIndex+NumToSkip;
3816 if (
N->isMachineOpcode()) {
3823 if (
N->isTargetOpcode())
3824 return N->isTargetStrictFPOpcode();
3825 return N->isStrictFPOpcode();
3830 auto *
C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
3835 if (
auto *FN = dyn_cast<FrameIndexSDNode>(
N->getOperand(0))) {
3838 int32_t Off =
C->getSExtValue();
3841 return (Off >= 0) && (((
A.value() - 1) & Off) ==
unsigned(Off));
3846void SelectionDAGISel::CannotYetSelect(
SDNode *
N) {
3849 Msg <<
"Cannot select: ";
3855 Msg <<
"\nIn function: " <<
MF->
getName();
3857 bool HasInputChain =
N->getOperand(0).getValueType() ==
MVT::Other;
3859 cast<ConstantSDNode>(
N->getOperand(HasInputChain))->getZExtValue();
3860 if (iid < Intrinsic::num_intrinsics)
3863 Msg <<
"target intrinsic %" <<
TII->
getName(iid);
3865 Msg <<
"unknown intrinsic #" << iid;
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
amdgpu AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
BlockVerifier::State From
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_ATTRIBUTE_ALWAYS_INLINE
LLVM_ATTRIBUTE_ALWAYS_INLINE - On compilers where we have a directive to do so, mark a method "always...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
This file defines the FastISel class.
mir Rename Register Operands
Machine Instruction Scheduler
unsigned const TargetRegisterInfo * TRI
typename CallsiteContextGraph< DerivedCCG, FuncTy, CallTy >::FuncInfo FuncInfo
print must be executed print the must be executed context for all instructions
const char LLVMTargetMachineRef TM
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
static cl::opt< bool > ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed"))
static cl::opt< bool > ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the post " "legalize types dag combine pass"))
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes, unsigned ChildNo)
CheckChildSame - Implements OP_CheckChildXSame.
static uint64_t decodeSignRotatedValue(uint64_t V)
Decode a signed value stored with the sign bit in the LSB for dense VBR encoding.
static cl::opt< bool > ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected"))
static LLVM_ATTRIBUTE_ALWAYS_INLINE uint64_t GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx)
GetVBR - decode a vbr encoding whose top bit is set.
static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F, MachineModuleInfo &MMI)
static void reportFastISelFailure(MachineFunction &MF, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R, bool ShouldAbort)
static void processDbgDeclare(FunctionLoweringInfo &FuncInfo, const Value *Address, DIExpression *Expr, DILocalVariable *Var, DebugLoc DbgLoc)
static cl::opt< bool > ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the second " "dag combine pass"))
static RegisterScheduler defaultListDAGScheduler("default", "Best scheduler for the target", createDefaultScheduler)
static unsigned IsPredicateKnownToFail(const unsigned char *Table, unsigned Index, SDValue N, bool &Result, const SelectionDAGISel &SDISel, SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
IsPredicateKnownToFail - If we know how and can do so without pushing a scope, evaluate the current n...
static cl::opt< int > EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction selection " "fails to lower an instruction: 0 disable the abort, 1 will " "abort but for args, calls and terminators, 2 will also " "abort for argument lowering, and 3 will never fallback " "to SelectionDAG."))
static void mapWasmLandingPadIndex(MachineBasicBlock *MBB, const CatchPadInst *CPI)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel, SDNode *N)
CheckNodePredicate - Implements OP_CheckNodePredicate.
static void processSingleLocVars(FunctionLoweringInfo &FuncInfo, FunctionVarLocs const *FnVarLocs)
Collect single location variable information generated with assignment tracking.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
static cl::opt< bool > UseMBPI("use-mbpi", cl::desc("use Machine Branch Probability Info"), cl::init(true), cl::Hidden)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
CheckSame - Implements OP_CheckSame.
static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse, bool IgnoreChains)
findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path beyond "ImmedUse".
static cl::opt< bool > ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the first " "dag combine pass"))
static cl::opt< bool > ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed"))
static void processDbgDeclares(FunctionLoweringInfo &FuncInfo)
Collect llvm.dbg.declare information.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL, unsigned ChildNo)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel)
CheckPatternPredicate - Implements OP_CheckPatternPredicate.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
static cl::opt< bool > ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize"))
static cl::opt< bool > ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize types"))
static SDNode * findGlueUse(SDNode *N)
findGlueUse - Return use of MVT::Glue value produced by the specified SDNode.
static cl::opt< RegisterScheduler::FunctionPassCtor, false, RegisterPassParser< RegisterScheduler > > ISHeuristic("pre-RA-sched", cl::init(&createDefaultScheduler), cl::Hidden, cl::desc("Instruction schedulers available (before register" " allocation):"))
ISHeuristic command line option for instruction schedulers.
static cl::opt< bool > EnableFastISelFallbackReport("fast-isel-report-on-fallback", cl::Hidden, cl::desc("Emit a diagnostic when \"fast\" instruction selection " "falls back to SelectionDAG."))
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, unsigned ChildNo)
static cl::opt< std::string > FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, cl::desc("Only display the basic block whose name " "matches this for all view-*-dags options"))
static SDValue HandleMergeInputChains(SmallVectorImpl< SDNode * > &ChainNodesMatched, SelectionDAG *CurDAG)
HandleMergeInputChains - This implements the OPC_EmitMergeInputChains operation for when the pattern ...
static bool isFoldedOrDeadInstruction(const Instruction *I, const FunctionLoweringInfo &FuncInfo)
isFoldedOrDeadInstruction - Return true if the specified instruction is side-effect free and is eithe...
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file describes how to lower LLVM code to machine code.
DEMANGLE_DUMP_METHOD void dump() const
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
Class for arbitrary precision integers.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
An immutable pass that tracks lazily created AssumptionCache objects.
LLVM Basic Block Representation.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
InstListType::const_iterator const_iterator
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
bool isEHPad() const
Return true if this basic block is an exception handling block.
const Instruction * getFirstMayFaultInst() const
Returns the first potential AsynchEH faulty instruction currently it checks for loads/stores (which m...
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Legacy analysis pass which computes BranchProbabilityInfo.
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
This is an important base class in LLVM.
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
A parsed version of the target data layout string in and methods for querying it.
This represents the llvm.dbg.declare instruction.
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for ISel fallback path.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
void setLastLocalValue(MachineInstr *I)
Update the position of the last instruction emitted for materializing constants for use in the curren...
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We're checking to see if we can fold LI into FoldInst.
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
void finishBasicBlock()
Flush the local value map.
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
unsigned arg_size() const
arg_size - Return the number of funcletpad arguments.
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th funcletpad argument.
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Data structure describing the variable locations in a function.
const VarLocInfo * single_locs_begin() const
DILocalVariable * getDILocalVariable(const VarLocInfo *Loc) const
Return the DILocalVariable for the location definition represented by ID.
const VarLocInfo * single_locs_end() const
One past the last single-location variable location definition.
const BasicBlock & getEntryBlock() const
DISubprogram * getSubprogram() const
Get the attached subprogram.
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool callsFunctionThatReturnsTwice() const
callsFunctionThatReturnsTwice - Return true if the function has a call to setjmp or other function th...
An analysis pass which caches information about the entire Module.
Module * getParent()
Get the module that this global value is contained inside of...
PointerType * getType() const
Global values are always pointers.
This class is used to form a handle around another node that is persistent and is updated across invo...
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
static bool isMemKind(unsigned Flag)
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag.
static unsigned getFlagWordForMem(unsigned InputFlag, unsigned Constraint)
Augment an existing flag word returned by getFlagWord with the constraint code for a memory constrain...
static bool isFuncKind(unsigned Flag)
static unsigned getMemoryConstraintID(unsigned Flag)
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
A wrapper class for inspecting calls to intrinsic functions.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
static void getLazyBFIAnalysisUsage(AnalysisUsage &AU)
Helper for client passes to set up the analysis usage on behalf of this pass.
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Describe properties that are true of each instruction in the target description file.
bool mayStore() const
Return true if this instruction could possibly modify memory.
bool mayLoad() const
Return true if this instruction could possibly read memory.
bool mayRaiseFPException() const
Return true if this instruction may raise a floating-point exception.
bool isCall() const
Return true if the instruction is a call.
bool isReturn() const
Return true if the instruction is a return.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
const MDNode * getMD() const
const MDOperand & getOperand(unsigned I) const
StringRef getString() const
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
instr_iterator instr_end()
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasCalls() const
Return true if the current function has any function calls.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
bool hasProperty(Property P) const
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setExposesReturnsTwice(bool B)
setCallsSetJmp - Set a flag that indicates if there's a call to a "returns twice" function.
void setHasInlineAsm(bool B)
Set a flag that indicates that the function contains inline assembly.
void setWasmLandingPadIndex(const MachineBasicBlock *LPad, unsigned Index)
Map the landing pad to its index. Used for Wasm exception handling.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
bool hasInlineAsm() const
Returns true if the function contains any inline assembly.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void finalizeDebugInstrRefs()
Finalise any partially emitted debug instructions.
void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef< unsigned > Sites)
Map the landing pad's EH symbol to the call site indexes.
void setUseDebugInstrRef(bool UseInstrRef)
Set whether this function will use instruction referencing or not.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad, and extract the exception handling information from the landingpad instruction...
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineModuleInfo & getMMI() const
bool shouldUseDebugInstrRef() const
Determine whether, in the current machine configuration, we should use instruction referencing or not...
const MachineFunctionProperties & getProperties() const
Get the function properties.
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable.
const MachineBasicBlock & front() const
void print(raw_ostream &OS, const SlotIndexes *=nullptr) const
print - Print out the MachineFunction in a format suitable for debugging to the specified stream.
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
const MachineBasicBlock * getParent() const
bool isDebugValue() const
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
This class contains meta information specific to a module.
const MCContext & getContext() const
bool usesMSVCFloatingPoint() const
void setUsesMSVCFloatingPoint(bool b)
Register getReg() const
getReg - Returns the register number.
MachinePassRegistry - Track the registration of machine passes.
defusechain_iterator - This class provides iterator support for machine operands in the function that...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
void EmitLiveInCopies(MachineBasicBlock *EntryMBB, const TargetRegisterInfo &TRI, const TargetInstrInfo &TII)
EmitLiveInCopies - Emit copies to initialize livein virtual registers into the given entry block.
use_instr_iterator use_instr_begin(Register RegNo) const
ArrayRef< std::pair< MCRegister, Register > > liveins() const
static use_instr_iterator use_instr_end()
void addPhysRegsUsedFromRegMask(const uint32_t *RegMask)
addPhysRegsUsedFromRegMask - Mark any registers not in RegMask as used.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
This class is used by SelectionDAGISel to temporarily override the optimization level on a per-functi...
OptLevelChanger(SelectionDAGISel &ISel, CodeGenOpt::Level NewOptLevel)
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
RegisterPassParser class - Handle the addition of new machine passes.
ScheduleDAGSDNodes *(*)(SelectionDAGISel *, CodeGenOpt::Level) FunctionPassCtor
static MachinePassRegistry< FunctionPassCtor > Registry
RegisterScheduler class - Track the registration of instruction schedulers.
Wrapper class representing virtual and physical registers.
static unsigned virtReg2Index(Register Reg)
Convert a virtual register number to a 0-based index.
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
void setNodeId(int Id)
Set unique node id.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckOrMask - The isel is trying to match something like (or X, 255).
CodeGenOpt::Level OptLevel
const TargetLowering * TLI
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
virtual bool CheckNodePredicate(SDNode *N, unsigned PredNo) const
CheckNodePredicate - This function is generated by tblgen in the target.
std::unique_ptr< OptimizationRemarkEmitter > ORE
Current optimization remark emitter.
MachineRegisterInfo * RegInfo
unsigned DAGSize
DAGSize - Size of DAG being instruction selected.
@ OPC_CheckChild2CondCode