65 #define DEBUG_TYPE "isel"
67 STATISTIC(NumFastIselFailures,
"Number of instructions fast isel failed on");
68 STATISTIC(NumFastIselSuccess,
"Number of instructions fast isel selected");
69 STATISTIC(NumFastIselBlocks,
"Number of blocks selected entirely by fast isel");
70 STATISTIC(NumDAGBlocks,
"Number of blocks selected using DAG");
71 STATISTIC(NumDAGIselRetries,
"Number of times dag isel has to try another path");
72 STATISTIC(NumEntryBlocks,
"Number of entry blocks encountered");
74 "Number of entry blocks where fast isel failed to lower arguments");
79 cl::desc(
"Enable extra verbose messages in the \"fast\" "
80 "instruction selector"));
83 STATISTIC(NumFastIselFailRet,
"Fast isel fails on Ret");
84 STATISTIC(NumFastIselFailBr,
"Fast isel fails on Br");
85 STATISTIC(NumFastIselFailSwitch,
"Fast isel fails on Switch");
86 STATISTIC(NumFastIselFailIndirectBr,
"Fast isel fails on IndirectBr");
87 STATISTIC(NumFastIselFailInvoke,
"Fast isel fails on Invoke");
88 STATISTIC(NumFastIselFailResume,
"Fast isel fails on Resume");
89 STATISTIC(NumFastIselFailUnreachable,
"Fast isel fails on Unreachable");
92 STATISTIC(NumFastIselFailAdd,
"Fast isel fails on Add");
93 STATISTIC(NumFastIselFailFAdd,
"Fast isel fails on FAdd");
94 STATISTIC(NumFastIselFailSub,
"Fast isel fails on Sub");
95 STATISTIC(NumFastIselFailFSub,
"Fast isel fails on FSub");
96 STATISTIC(NumFastIselFailMul,
"Fast isel fails on Mul");
97 STATISTIC(NumFastIselFailFMul,
"Fast isel fails on FMul");
98 STATISTIC(NumFastIselFailUDiv,
"Fast isel fails on UDiv");
99 STATISTIC(NumFastIselFailSDiv,
"Fast isel fails on SDiv");
100 STATISTIC(NumFastIselFailFDiv,
"Fast isel fails on FDiv");
101 STATISTIC(NumFastIselFailURem,
"Fast isel fails on URem");
102 STATISTIC(NumFastIselFailSRem,
"Fast isel fails on SRem");
103 STATISTIC(NumFastIselFailFRem,
"Fast isel fails on FRem");
106 STATISTIC(NumFastIselFailAnd,
"Fast isel fails on And");
107 STATISTIC(NumFastIselFailOr,
"Fast isel fails on Or");
108 STATISTIC(NumFastIselFailXor,
"Fast isel fails on Xor");
111 STATISTIC(NumFastIselFailAlloca,
"Fast isel fails on Alloca");
112 STATISTIC(NumFastIselFailLoad,
"Fast isel fails on Load");
113 STATISTIC(NumFastIselFailStore,
"Fast isel fails on Store");
114 STATISTIC(NumFastIselFailAtomicCmpXchg,
"Fast isel fails on AtomicCmpXchg");
115 STATISTIC(NumFastIselFailAtomicRMW,
"Fast isel fails on AtomicRWM");
116 STATISTIC(NumFastIselFailFence,
"Fast isel fails on Frence");
117 STATISTIC(NumFastIselFailGetElementPtr,
"Fast isel fails on GetElementPtr");
120 STATISTIC(NumFastIselFailTrunc,
"Fast isel fails on Trunc");
121 STATISTIC(NumFastIselFailZExt,
"Fast isel fails on ZExt");
122 STATISTIC(NumFastIselFailSExt,
"Fast isel fails on SExt");
123 STATISTIC(NumFastIselFailFPTrunc,
"Fast isel fails on FPTrunc");
124 STATISTIC(NumFastIselFailFPExt,
"Fast isel fails on FPExt");
125 STATISTIC(NumFastIselFailFPToUI,
"Fast isel fails on FPToUI");
126 STATISTIC(NumFastIselFailFPToSI,
"Fast isel fails on FPToSI");
127 STATISTIC(NumFastIselFailUIToFP,
"Fast isel fails on UIToFP");
128 STATISTIC(NumFastIselFailSIToFP,
"Fast isel fails on SIToFP");
129 STATISTIC(NumFastIselFailIntToPtr,
"Fast isel fails on IntToPtr");
130 STATISTIC(NumFastIselFailPtrToInt,
"Fast isel fails on PtrToInt");
131 STATISTIC(NumFastIselFailBitCast,
"Fast isel fails on BitCast");
134 STATISTIC(NumFastIselFailICmp,
"Fast isel fails on ICmp");
135 STATISTIC(NumFastIselFailFCmp,
"Fast isel fails on FCmp");
136 STATISTIC(NumFastIselFailPHI,
"Fast isel fails on PHI");
137 STATISTIC(NumFastIselFailSelect,
"Fast isel fails on Select");
138 STATISTIC(NumFastIselFailCall,
"Fast isel fails on Call");
139 STATISTIC(NumFastIselFailShl,
"Fast isel fails on Shl");
140 STATISTIC(NumFastIselFailLShr,
"Fast isel fails on LShr");
141 STATISTIC(NumFastIselFailAShr,
"Fast isel fails on AShr");
142 STATISTIC(NumFastIselFailVAArg,
"Fast isel fails on VAArg");
143 STATISTIC(NumFastIselFailExtractElement,
"Fast isel fails on ExtractElement");
144 STATISTIC(NumFastIselFailInsertElement,
"Fast isel fails on InsertElement");
145 STATISTIC(NumFastIselFailShuffleVector,
"Fast isel fails on ShuffleVector");
146 STATISTIC(NumFastIselFailExtractValue,
"Fast isel fails on ExtractValue");
147 STATISTIC(NumFastIselFailInsertValue,
"Fast isel fails on InsertValue");
148 STATISTIC(NumFastIselFailLandingPad,
"Fast isel fails on LandingPad");
151 STATISTIC(NumFastIselFailIntrinsicCall,
"Fast isel fails on Intrinsic call");
152 STATISTIC(NumFastIselFailSAddWithOverflow,
153 "Fast isel fails on sadd.with.overflow");
154 STATISTIC(NumFastIselFailUAddWithOverflow,
155 "Fast isel fails on uadd.with.overflow");
156 STATISTIC(NumFastIselFailSSubWithOverflow,
157 "Fast isel fails on ssub.with.overflow");
158 STATISTIC(NumFastIselFailUSubWithOverflow,
159 "Fast isel fails on usub.with.overflow");
160 STATISTIC(NumFastIselFailSMulWithOverflow,
161 "Fast isel fails on smul.with.overflow");
162 STATISTIC(NumFastIselFailUMulWithOverflow,
163 "Fast isel fails on umul.with.overflow");
164 STATISTIC(NumFastIselFailFrameaddress,
"Fast isel fails on Frameaddress");
165 STATISTIC(NumFastIselFailSqrt,
"Fast isel fails on sqrt call");
166 STATISTIC(NumFastIselFailStackMap,
"Fast isel fails on StackMap call");
167 STATISTIC(NumFastIselFailPatchPoint,
"Fast isel fails on PatchPoint call");
172 cl::desc(
"Enable verbose messages in the \"fast\" "
173 "instruction selector"));
176 cl::desc(
"Enable abort calls when \"fast\" instruction selection "
177 "fails to lower an instruction: 0 disable the abort, 1 will "
178 "abort but for args, calls and terminators, 2 will also "
179 "abort for argument lowering, and 3 will never fallback "
180 "to SelectionDAG."));
184 cl::desc(
"use Machine Branch Probability Info"),
190 cl::desc(
"Only display the basic block whose name "
191 "matches this for all view-*-dags options"));
194 cl::desc(
"Pop up a window to show dags before the first "
195 "dag combine pass"));
198 cl::desc(
"Pop up a window to show dags before legalize types"));
201 cl::desc(
"Pop up a window to show dags before legalize"));
204 cl::desc(
"Pop up a window to show dags before the second "
205 "dag combine pass"));
208 cl::desc(
"Pop up a window to show dags before the post legalize types"
209 " dag combine pass"));
212 cl::desc(
"Pop up a window to show isel dags as they are selected"));
215 cl::desc(
"Pop up a window to show sched dags as they are processed"));
218 cl::desc(
"Pop up a window to show SUnit dags after they are processed"));
244 cl::desc(
"Instruction schedulers available (before register"
264 if (NewOptLevel == SavedOptLevel)
268 DEBUG(
dbgs() <<
"\nChanging optimization level for Function "
270 DEBUG(
dbgs() <<
"\tBefore: -O" << SavedOptLevel
271 <<
" ; After: -O" << NewOptLevel <<
"\n");
284 DEBUG(
dbgs() <<
"\nRestoring optimization level for Function "
287 <<
" ; After: -O" << SavedOptLevel <<
"\n");
304 return SchedulerCtor(IS, OptLevel);
318 "Unknown sched type!");
336 dbgs() <<
"If a target marks an instruction with "
337 "'usesCustomInserter', it must implement "
338 "TargetLowering::EmitInstrWithCustomInserter!";
346 "If a target marks an instruction with 'hasPostISelHook', "
347 "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
410 if (!CE || !CE->
canTrap())
continue;
435 "-fast-isel-verbose requires -fast-isel");
437 "-fast-isel-abort > 0 requires -fast-isel");
457 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
458 LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
459 GFI = Fn.
hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) :
nullptr;
469 FuncInfo->
BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
490 if (isa<UnreachableInst>(Term) || isa<ReturnInst>(Term))
504 SelectAllBasicBlocks(Fn);
521 if (Term !=
MBB.
end() && Term->isReturn()) {
534 LiveInMap.
insert(std::make_pair(LI->first, LI->second));
551 DEBUG(
dbgs() <<
"Dropping debug info for dead vreg"
557 if (LDI != LiveInMap.
end()) {
558 assert(!hasFI &&
"There's no handling of frame pointer updating here yet "
567 assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
568 "Expected inlined-at fields to agree");
570 BuildMI(*EntryMBB, ++InsertPos, DL,
TII->
get(TargetOpcode::DBG_VALUE),
571 IsIndirect, LDI->second,
Offset, Variable, Expr);
583 CopyUseMI =
UseMI;
continue;
586 CopyUseMI =
nullptr;
break;
593 CopyUseMI->getOperand(0).getReg(),
Offset, Variable, Expr);
602 for (
const auto &
MBB : *
MF) {
603 if (MFI.
hasCalls() && MF->hasInlineAsm())
606 for (
const auto &
MI :
MBB) {
609 MI.isStackAligningInlineAsm()) {
612 if (
MI.isInlineAsm()) {
613 MF->setHasInlineAsm(
true);
627 unsigned From =
I->first;
628 unsigned To =
I->second;
663 DEBUG(
dbgs() <<
"*** MachineFunction at end of ISel ***\n");
686 void SelectionDAGISel::ComputeLiveOutVRegInfo() {
699 if (!VisitedNodes.
insert(N).second)
724 }
while (!Worklist.
empty());
727 void SelectionDAGISel::CodeGenAndEmitDAG() {
729 StringRef GroupDescription =
"Instruction Selection and Scheduling";
730 std::string BlockName;
731 int BlockNumber = -1;
733 bool MatchFilterBB =
false; (void)MatchFilterBB;
749 DEBUG(
dbgs() <<
"Initial selection DAG: BB#" << BlockNumber
750 <<
" '" << BlockName <<
"'\n";
CurDAG->
dump());
762 DEBUG(
dbgs() <<
"Optimized lowered selection DAG: BB#" << BlockNumber
763 <<
" '" << BlockName <<
"'\n";
CurDAG->
dump());
777 DEBUG(
dbgs() <<
"Type-legalized selection DAG: BB#" << BlockNumber
778 <<
" '" << BlockName <<
"'\n";
CurDAG->
dump());
793 DEBUG(
dbgs() <<
"Optimized type-legalized selection DAG: BB#" << BlockNumber
794 <<
" '" << BlockName <<
"'\n";
CurDAG->
dump());
821 DEBUG(
dbgs() <<
"Optimized vector-legalized selection DAG: BB#"
822 << BlockNumber <<
" '" << BlockName <<
"'\n";
CurDAG->
dump());
834 DEBUG(
dbgs() <<
"Legalized selection DAG: BB#" << BlockNumber
835 <<
" '" << BlockName <<
"'\n";
CurDAG->
dump());
847 DEBUG(
dbgs() <<
"Optimized legalized selection DAG: BB#" << BlockNumber
848 <<
" '" << BlockName <<
"'\n";
CurDAG->
dump());
851 ComputeLiveOutVRegInfo();
861 DoInstructionSelection();
864 DEBUG(
dbgs() <<
"Selected selection DAG: BB#" << BlockNumber
865 <<
" '" << BlockName <<
"'\n";
CurDAG->
dump());
895 if (FirstMBB != LastMBB)
916 :
SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
928 void SelectionDAGISel::DoInstructionSelection() {
929 DEBUG(
dbgs() <<
"===== Instruction selection begins: BB#"
949 ISelUpdater ISU(*
CurDAG, ISelPosition);
956 SDNode *Node = &*--ISelPosition;
969 DEBUG(
dbgs() <<
"===== Instruction selection ends:\n");
976 if (
const IntrinsicInst *EHPtrCall = dyn_cast<IntrinsicInst>(U)) {
978 if (IID == Intrinsic::eh_exceptionpointer ||
979 IID == Intrinsic::eh_exceptioncode)
988 bool SelectionDAGISel::PrepareEHLandingPad() {
997 if (
const auto *CPI = dyn_cast<CatchPadInst>(LLVMBB->
getFirstNonPHI())) {
1002 assert(EHPhysReg &&
"target lacks exception pointer register");
1006 TII->
get(TargetOpcode::COPY), VReg)
1043 !isa<TerminatorInst>(
I) &&
1044 !isa<DbgInfoIntrinsic>(I) &&
1056 default:
assert (0 &&
"<Invalid operator> ");
1060 case Instruction::Br: NumFastIselFailBr++;
return;
1062 case Instruction::IndirectBr: NumFastIselFailIndirectBr++;
return;
1063 case Instruction::Invoke: NumFastIselFailInvoke++;
return;
1064 case Instruction::Resume: NumFastIselFailResume++;
return;
1065 case Instruction::Unreachable: NumFastIselFailUnreachable++;
return;
1069 case Instruction::FAdd: NumFastIselFailFAdd++;
return;
1070 case Instruction::Sub: NumFastIselFailSub++;
return;
1071 case Instruction::FSub: NumFastIselFailFSub++;
return;
1072 case Instruction::Mul: NumFastIselFailMul++;
return;
1073 case Instruction::FMul: NumFastIselFailFMul++;
return;
1074 case Instruction::UDiv: NumFastIselFailUDiv++;
return;
1075 case Instruction::SDiv: NumFastIselFailSDiv++;
return;
1076 case Instruction::FDiv: NumFastIselFailFDiv++;
return;
1077 case Instruction::URem: NumFastIselFailURem++;
return;
1078 case Instruction::SRem: NumFastIselFailSRem++;
return;
1079 case Instruction::FRem: NumFastIselFailFRem++;
return;
1087 case Instruction::Alloca: NumFastIselFailAlloca++;
return;
1090 case Instruction::AtomicCmpXchg: NumFastIselFailAtomicCmpXchg++;
return;
1091 case Instruction::AtomicRMW: NumFastIselFailAtomicRMW++;
return;
1092 case Instruction::Fence: NumFastIselFailFence++;
return;
1093 case Instruction::GetElementPtr: NumFastIselFailGetElementPtr++;
return;
1096 case Instruction::Trunc: NumFastIselFailTrunc++;
return;
1097 case Instruction::ZExt: NumFastIselFailZExt++;
return;
1098 case Instruction::SExt: NumFastIselFailSExt++;
return;
1099 case Instruction::FPTrunc: NumFastIselFailFPTrunc++;
return;
1100 case Instruction::FPExt: NumFastIselFailFPExt++;
return;
1101 case Instruction::FPToUI: NumFastIselFailFPToUI++;
return;
1102 case Instruction::FPToSI: NumFastIselFailFPToSI++;
return;
1103 case Instruction::UIToFP: NumFastIselFailUIToFP++;
return;
1104 case Instruction::SIToFP: NumFastIselFailSIToFP++;
return;
1105 case Instruction::IntToPtr: NumFastIselFailIntToPtr++;
return;
1106 case Instruction::PtrToInt: NumFastIselFailPtrToInt++;
return;
1107 case Instruction::BitCast: NumFastIselFailBitCast++;
return;
1110 case Instruction::ICmp: NumFastIselFailICmp++;
return;
1111 case Instruction::FCmp: NumFastIselFailFCmp++;
return;
1112 case Instruction::PHI: NumFastIselFailPHI++;
return;
1115 if (
auto const *Intrinsic = dyn_cast<IntrinsicInst>(I)) {
1116 switch (Intrinsic->getIntrinsicID()) {
1118 NumFastIselFailIntrinsicCall++;
return;
1119 case Intrinsic::sadd_with_overflow:
1120 NumFastIselFailSAddWithOverflow++;
return;
1121 case Intrinsic::uadd_with_overflow:
1122 NumFastIselFailUAddWithOverflow++;
return;
1123 case Intrinsic::ssub_with_overflow:
1124 NumFastIselFailSSubWithOverflow++;
return;
1125 case Intrinsic::usub_with_overflow:
1126 NumFastIselFailUSubWithOverflow++;
return;
1127 case Intrinsic::smul_with_overflow:
1128 NumFastIselFailSMulWithOverflow++;
return;
1129 case Intrinsic::umul_with_overflow:
1130 NumFastIselFailUMulWithOverflow++;
return;
1131 case Intrinsic::frameaddress:
1132 NumFastIselFailFrameaddress++;
return;
1133 case Intrinsic::sqrt:
1134 NumFastIselFailSqrt++;
return;
1135 case Intrinsic::experimental_stackmap:
1136 NumFastIselFailStackMap++;
return;
1137 case Intrinsic::experimental_patchpoint_void:
1138 case Intrinsic::experimental_patchpoint_i64:
1139 NumFastIselFailPatchPoint++;
return;
1142 NumFastIselFailCall++;
1145 case Instruction::Shl: NumFastIselFailShl++;
return;
1146 case Instruction::LShr: NumFastIselFailLShr++;
return;
1147 case Instruction::AShr: NumFastIselFailAShr++;
return;
1148 case Instruction::VAArg: NumFastIselFailVAArg++;
return;
1149 case Instruction::ExtractElement: NumFastIselFailExtractElement++;
return;
1150 case Instruction::InsertElement: NumFastIselFailInsertElement++;
return;
1151 case Instruction::ShuffleVector: NumFastIselFailShuffleVector++;
return;
1152 case Instruction::ExtractValue: NumFastIselFailExtractValue++;
return;
1153 case Instruction::InsertValue: NumFastIselFailInsertValue++;
return;
1154 case Instruction::LandingPad: NumFastIselFailLandingPad++;
return;
1172 bool HaveSeenSwiftErrorArg =
false;
1175 if (AI->hasSwiftErrorAttr()) {
1176 assert(!HaveSeenSwiftErrorArg &&
1177 "Must have only one swifterror parameter");
1178 (void)HaveSeenSwiftErrorArg;
1179 HaveSeenSwiftErrorArg =
true;
1184 for (
const auto &LLVMBB : Fn)
1185 for (
const auto &Inst : LLVMBB) {
1186 if (
const AllocaInst *Alloca = dyn_cast<AllocaInst>(&Inst))
1187 if (Alloca->isSwiftError())
1226 auto *TLI = FuncInfo->
TLI;
1227 if (!TLI->supportSwiftError())
1245 auto Key = std::make_pair(MBB, SwiftErrorVal);
1249 unsigned UUseVReg = UpwardsUse ? UUseIt->second : 0;
1251 assert(!(UpwardsUse && !DownwardDef) &&
1252 "We can't have an upwards use but no downwards def");
1257 if (!UpwardsUse && DownwardDef)
1268 if (!Visited.
insert(Pred).second)
1281 UUseVReg = UUseIt->second;
1288 VRegs.
size() >= 1 &&
1291 [&](
const std::pair<const MachineBasicBlock *, unsigned> &V)
1292 ->
bool {
return V.second != VRegs[0].second; }) !=
1297 if (!UpwardsUse && !needPHI) {
1299 "No predecessors? The entry block should bail out earlier");
1305 auto DLoc = isa<Instruction>(SwiftErrorVal)
1306 ? dyn_cast<Instruction>(SwiftErrorVal)->getDebugLoc()
1313 unsigned DestReg = UUseVReg;
1316 .addReg(VRegs[0].second);
1323 auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL));
1325 UpwardsUse ? UUseVReg
1329 TII->get(TargetOpcode::PHI), PHIVReg);
1330 for (
auto BBRegPair : VRegs) {
1331 SwiftErrorPHI.
addReg(BBRegPair.second).
addMBB(BBRegPair.first);
1342 void SelectionDAGISel::SelectAllBasicBlocks(
const Function &Fn) {
1353 I = RPOT.begin(),
E = RPOT.end();
I !=
E; ++
I) {
1357 bool AllPredsVisited =
true;
1361 AllPredsVisited =
false;
1366 if (AllPredsVisited) {
1394 if (!PrepareEHLandingPad())
1409 ++NumFastIselFailLowerArguments;
1417 CodeGenAndEmitDAG();
1429 unsigned NumFastIselRemaining = std::distance(Begin, End);
1431 for (; BI != Begin; --BI) {
1436 --NumFastIselRemaining;
1446 --NumFastIselRemaining;
1447 ++NumFastIselSuccess;
1452 while (BeforeInst != &*Begin) {
1457 if (BeforeInst != Inst && isa<LoadInst>(BeforeInst) &&
1462 --NumFastIselRemaining;
1463 ++NumFastIselSuccess;
1474 if (isa<CallInst>(Inst)) {
1477 dbgs() <<
"FastISel missed call: ";
1492 bool HadTailCall =
false;
1494 SelectBasicBlock(Inst->
getIterator(), BI, HadTailCall);
1506 unsigned RemainingNow = std::distance(Begin, BI);
1507 NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1508 NumFastIselRemaining = RemainingNow;
1514 if (isa<TerminatorInst>(Inst)) {
1516 dbgs() <<
"FastISel missed terminator: ";
1520 dbgs() <<
"FastISel miss: ";
1529 NumFastIselFailures += NumFastIselRemaining;
1541 if (getAnalysis<StackProtector>().shouldEmitSDCheck(*LLVMBB)) {
1542 bool FunctionBasedInstrumentation =
1545 FunctionBasedInstrumentation);
1551 ++NumFastIselBlocks;
1558 SelectBasicBlock(Begin, BI, HadTailCall);
1610 &&
"Should have a copy implying we should have 2 arguments.");
1614 if (!OPI2->
isReg() ||
1639 if (SplitPoint == BB->
begin())
1647 SplitPoint = Previous;
1648 if (Previous == Start)
1657 SelectionDAGISel::FinishBasicBlock() {
1658 DEBUG(
dbgs() <<
"Total amount of phi nodes to update: "
1661 dbgs() <<
"Node " <<
i <<
" : ("
1670 "This is not a machine PHI node that we are updating!");
1689 CodeGenAndEmitDAG();
1707 SuccessMBB->
splice(SuccessMBB->
end(), ParentMBB,
1717 CodeGenAndEmitDAG();
1721 if (FailureMBB->
empty()) {
1727 CodeGenAndEmitDAG();
1745 CodeGenAndEmitDAG();
1749 for (
unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
1750 UnhandledProb -= BTB.Cases[j].ExtraProb;
1765 if (BTB.ContiguousRange && j + 2 == ej) {
1768 NextMBB = BTB.Cases[j + 1].TargetBB;
1769 }
else if (j + 1 == ej) {
1771 NextMBB = BTB.Default;
1774 NextMBB = BTB.Cases[j + 1].ThisBB;
1782 CodeGenAndEmitDAG();
1784 if (BTB.ContiguousRange && j + 2 == ej) {
1786 BTB.Cases.pop_back();
1797 "This is not a machine PHI node that we are updating!");
1800 if (PHIBB == BTB.Default) {
1802 if (!BTB.ContiguousRange) {
1804 .addMBB(BTB.Cases.back().ThisBB);
1808 for (
unsigned j = 0, ej = BTB.Cases.size();
1821 for (
unsigned i = 0, e =
SDB->
JTCases.size();
i != e; ++
i) {
1832 CodeGenAndEmitDAG();
1842 CodeGenAndEmitDAG();
1850 "This is not a machine PHI node that we are updating!");
1879 CodeGenAndEmitDAG();
1889 for (
unsigned i = 0, e = Succs.
size();
i != e; ++
i) {
1897 MBBI != MBBE && MBBI->isPHI(); ++MBBI) {
1900 for (
unsigned pn = 0; ; ++pn) {
1902 "Didn't find PHI entry!");
1933 int64_t DesiredMaskS)
const {
1938 if (ActualMask == DesiredMask)
1947 APInt NeededMask = DesiredMask & ~ActualMask;
1962 int64_t DesiredMaskS)
const {
1967 if (ActualMask == DesiredMask)
1976 APInt NeededMask = DesiredMask & ~ActualMask;
1978 APInt KnownZero, KnownOne;
1982 if ((NeededMask & KnownOne) == NeededMask)
1995 std::vector<SDValue> InOps;
2004 if (InOps[e-1].getValueType() ==
MVT::Glue)
2008 unsigned Flags = cast<ConstantSDNode>(InOps[
i])->getZExtValue();
2011 Ops.insert(Ops.end(), InOps.begin()+
i,
2016 "Memory operand with multiple values?");
2018 unsigned TiedToOperand;
2022 Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2023 for (; TiedToOperand; --TiedToOperand) {
2025 Flags = cast<ConstantSDNode>(InOps[CurOp])->getZExtValue();
2030 std::vector<SDValue> SelOps;
2041 Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());
2047 if (e != InOps.size())
2048 Ops.push_back(InOps.back());
2069 bool IgnoreChains) {
2083 if (!Visited.
insert(Use).second)
2093 if (Use == ImmedUse || Use == Root)
2100 if (
findNonImmUse(N, Def, ImmedUse, Root, Visited, IgnoreChains))
2118 bool IgnoreChains) {
2177 IgnoreChains =
false;
2185 void SelectionDAGISel::Select_INLINEASM(
SDNode *N) {
2198 void SelectionDAGISel::Select_READ_REGISTER(
SDNode *
Op) {
2212 void SelectionDAGISel::Select_WRITE_REGISTER(
SDNode *Op) {
2226 void SelectionDAGISel::Select_UNDEF(
SDNode *N) {
2232 GetVBR(uint64_t Val,
const unsigned char *MatcherTable,
unsigned &Idx) {
2233 assert(Val >= 128 &&
"Not a VBR");
2239 NextBits = MatcherTable[Idx++];
2240 Val |= (NextBits&127) << Shift;
2242 }
while (NextBits & 128);
2249 void SelectionDAGISel::UpdateChains(
2256 if (!ChainNodesMatched.
empty()) {
2258 "Matched input chains but didn't produce a chain");
2261 for (
unsigned i = 0, e = ChainNodesMatched.
size();
i != e; ++
i) {
2262 SDNode *ChainNode = ChainNodesMatched[
i];
2269 "Deleted node left in chain");
2273 if (ChainNode == NodeToMatch && isMorphNodeTo)
2282 std::replace(ChainNodesMatched.
begin(), ChainNodesMatched.
end(),
N,
2283 static_cast<SDNode *
>(
nullptr));
2288 if (ChainNode != NodeToMatch && ChainNode->
use_empty() &&
2294 if (!NowDeadNodes.
empty())
2297 DEBUG(
dbgs() <<
"ISEL: Match complete!\n");
2323 E = ChainedNode->
use_end(); UI !=
E; ++UI) {
2325 if (UI.getUse().getValueType() !=
MVT::Other)
continue;
2335 unsigned UserOpcode = User->
getOpcode();
2362 ChainedNodesInPattern.
end(), User))
2400 auto MemoizeResult = TokenFactorResult.
find(User);
2401 bool Visited = MemoizeResult != TokenFactorResult.
end();
2404 auto Res =
WalkChainUsers(User, ChainedNodesInPattern, TokenFactorResult,
2405 InteriorChainedNodes);
2406 MemoizeResult = TokenFactorResult.
insert(std::make_pair(User, Res)).first;
2408 switch (MemoizeResult->second) {
2453 for (
unsigned i = 0, e = ChainNodesMatched.
size();
i != e; ++
i) {
2463 for (
unsigned i = 0, e = ChainNodesMatched.
size();
i != e; ++
i) {
2466 SDNode *N = ChainNodesMatched[
i];
2472 SDValue InChain = ChainNodesMatched[
i]->getOperand(0);
2487 if (InputChains.
size() == 1)
2488 return InputChains[0];
2494 SDNode *SelectionDAGISel::
2503 int OldGlueResultNo = -1, OldChainResultNo = -1;
2507 OldGlueResultNo = NTMNumResults-1;
2508 if (NTMNumResults != 1 &&
2510 OldChainResultNo = NTMNumResults-2;
2512 OldChainResultNo = NTMNumResults-1;
2530 (
unsigned)OldGlueResultNo != ResNumResults-1)
2532 SDValue(Res, ResNumResults-1));
2534 if ((EmitNodeInfo & OPFL_GlueOutput) != 0)
2538 if ((EmitNodeInfo &
OPFL_Chain) && OldChainResultNo != -1 &&
2539 (
unsigned)OldChainResultNo != ResNumResults-1)
2541 SDValue(Res, ResNumResults-1));
2555 CheckSame(
const unsigned char *MatcherTable,
unsigned &MatcherIndex,
2559 unsigned RecNo = MatcherTable[MatcherIndex++];
2560 assert(RecNo < RecordedNodes.size() &&
"Invalid CheckSame");
2561 return N == RecordedNodes[RecNo].first;
2591 CheckOpcode(
const unsigned char *MatcherTable,
unsigned &MatcherIndex,
2593 uint16_t Opc = MatcherTable[MatcherIndex++];
2594 Opc |= (
unsigned short)MatcherTable[MatcherIndex++] << 8;
2621 return cast<CondCodeSDNode>(
N)->
get() ==
2629 if (cast<VTSDNode>(N)->getVT() == VT)
2639 int64_t Val = MatcherTable[MatcherIndex++];
2641 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2649 SDValue N,
unsigned ChildNo) {
2656 CheckAndImm(
const unsigned char *MatcherTable,
unsigned &MatcherIndex,
2658 int64_t Val = MatcherTable[MatcherIndex++];
2660 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2669 CheckOrImm(
const unsigned char *MatcherTable,
unsigned &MatcherIndex,
2671 int64_t Val = MatcherTable[MatcherIndex++];
2673 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
2692 switch (Table[Index++]) {
2697 Result = !
::CheckSame(Table, Index, N, RecordedNodes);
2767 unsigned NumRecordedNodes;
2770 unsigned NumMatchedMemRefs;
2773 SDValue InputChain, InputGlue;
2776 bool HasChainNodesMatched;
2792 :
SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
2793 RecordedNodes(
RN), MatchScopes(MS) {}
2804 if (N == *NodeToMatch)
2809 for (
auto &
I : RecordedNodes)
2810 if (
I.first.getNode() ==
N)
2813 for (
auto &
I : MatchScopes)
2814 for (
auto &J :
I.NodeStack)
2815 if (J.getNode() ==
N)
2822 const unsigned char *MatcherTable,
2823 unsigned TableSize) {
2859 Select_INLINEASM(NodeToMatch);
2862 Select_READ_REGISTER(NodeToMatch);
2865 Select_WRITE_REGISTER(NodeToMatch);
2868 Select_UNDEF(NodeToMatch);
2877 NodeStack.push_back(N);
2895 SDValue InputChain, InputGlue;
2903 DEBUG(
dbgs() <<
"ISEL: Starting pattern match on root node: ";
2911 unsigned MatcherIndex = 0;
2913 if (!OpcodeOffset.empty()) {
2915 if (N.
getOpcode() < OpcodeOffset.size())
2916 MatcherIndex = OpcodeOffset[N.
getOpcode()];
2917 DEBUG(
dbgs() <<
" Initial Opcode index to " << MatcherIndex <<
"\n");
2926 unsigned CaseSize = MatcherTable[Idx++];
2928 CaseSize =
GetVBR(CaseSize, MatcherTable, Idx);
2929 if (CaseSize == 0)
break;
2932 uint16_t Opc = MatcherTable[Idx++];
2933 Opc |= (
unsigned short)MatcherTable[Idx++] << 8;
2934 if (Opc >= OpcodeOffset.size())
2935 OpcodeOffset.resize((Opc+1)*2);
2936 OpcodeOffset[Opc] = Idx;
2941 if (N.
getOpcode() < OpcodeOffset.size())
2942 MatcherIndex = OpcodeOffset[N.
getOpcode()];
2946 assert(MatcherIndex < TableSize &&
"Invalid index");
2948 unsigned CurrentOpcodeIndex = MatcherIndex;
2961 unsigned NumToSkip = MatcherTable[MatcherIndex++];
2962 if (NumToSkip & 128)
2963 NumToSkip =
GetVBR(NumToSkip, MatcherTable, MatcherIndex);
2965 if (NumToSkip == 0) {
2970 FailIndex = MatcherIndex+NumToSkip;
2972 unsigned MatcherIndexOfPredicate = MatcherIndex;
2973 (void)MatcherIndexOfPredicate;
2980 Result, *
this, RecordedNodes);
2984 DEBUG(
dbgs() <<
" Skipped scope entry (due to false predicate) at "
2985 <<
"index " << MatcherIndexOfPredicate
2986 <<
", continuing at " << FailIndex <<
"\n");
2987 ++NumDAGIselRetries;
2991 MatcherIndex = FailIndex;
2995 if (FailIndex == 0)
break;
2999 MatchScope NewEntry;
3000 NewEntry.FailIndex = FailIndex;
3001 NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end());
3002 NewEntry.NumRecordedNodes = RecordedNodes.
size();
3003 NewEntry.NumMatchedMemRefs = MatchedMemRefs.
size();
3004 NewEntry.InputChain = InputChain;
3005 NewEntry.InputGlue = InputGlue;
3006 NewEntry.HasChainNodesMatched = !ChainNodesMatched.
empty();
3012 SDNode *Parent =
nullptr;
3013 if (NodeStack.size() > 1)
3014 Parent = NodeStack[NodeStack.size()-2].getNode();
3015 RecordedNodes.
push_back(std::make_pair(N, Parent));
3032 MatchedMemRefs.
push_back(cast<MemSDNode>(N)->getMemOperand());
3043 unsigned ChildNo = MatcherTable[MatcherIndex++];
3047 NodeStack.push_back(N);
3059 NodeStack.push_back(N);
3065 NodeStack.pop_back();
3066 assert(!NodeStack.empty() &&
"Node stack imbalance!");
3067 N = NodeStack.back();
3071 if (!::
CheckSame(MatcherTable, MatcherIndex, N, RecordedNodes))
break;
3076 if (!::
CheckChildSame(MatcherTable, MatcherIndex, N, RecordedNodes,
3090 unsigned CPNum = MatcherTable[MatcherIndex++];
3091 unsigned RecNo = MatcherTable[MatcherIndex++];
3092 assert(RecNo < RecordedNodes.
size() &&
"Invalid CheckComplexPat");
3096 std::unique_ptr<MatchStateUpdater> MSU;
3098 MSU.reset(
new MatchStateUpdater(*
CurDAG, &NodeToMatch, RecordedNodes,
3102 RecordedNodes[RecNo].first, CPNum,
3108 if (!::
CheckOpcode(MatcherTable, MatcherIndex, N.getNode()))
break;
3118 unsigned CurNodeOpcode = N.getOpcode();
3119 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3123 CaseSize = MatcherTable[MatcherIndex++];
3125 CaseSize =
GetVBR(CaseSize, MatcherTable, MatcherIndex);
3126 if (CaseSize == 0)
break;
3128 uint16_t Opc = MatcherTable[MatcherIndex++];
3129 Opc |= (
unsigned short)MatcherTable[MatcherIndex++] << 8;
3132 if (CurNodeOpcode == Opc)
3136 MatcherIndex += CaseSize;
3140 if (CaseSize == 0)
break;
3143 DEBUG(
dbgs() <<
" OpcodeSwitch from " << SwitchStart
3144 <<
" to " << MatcherIndex <<
"\n");
3149 MVT CurNodeVT = N.getSimpleValueType();
3150 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3154 CaseSize = MatcherTable[MatcherIndex++];
3156 CaseSize =
GetVBR(CaseSize, MatcherTable, MatcherIndex);
3157 if (CaseSize == 0)
break;
3164 if (CurNodeVT == CaseVT)
3168 MatcherIndex += CaseSize;
3172 if (CaseSize == 0)
break;
3175 DEBUG(
dbgs() <<
" TypeSwitch[" <<
EVT(CurNodeVT).getEVTString()
3176 <<
"] from " << SwitchStart <<
" to " << MatcherIndex<<
'\n');
3197 if (!::
CheckInteger(MatcherTable, MatcherIndex, N))
break;
3206 if (!::
CheckAndImm(MatcherTable, MatcherIndex, N, *
this))
break;
3209 if (!::
CheckOrImm(MatcherTable, MatcherIndex, N, *
this))
break;
3213 assert(NodeStack.size() != 1 &&
"No parent node");
3216 bool HasMultipleUses =
false;
3217 for (
unsigned i = 1, e = NodeStack.size()-1;
i != e; ++
i)
3218 if (!NodeStack[
i].hasOneUse()) {
3219 HasMultipleUses =
true;
3222 if (HasMultipleUses)
break;
3238 int64_t Val = MatcherTable[MatcherIndex++];
3240 Val =
GetVBR(Val, MatcherTable, MatcherIndex);
3241 RecordedNodes.
push_back(std::pair<SDValue, SDNode*>(
3249 unsigned RegNo = MatcherTable[MatcherIndex++];
3250 RecordedNodes.
push_back(std::pair<SDValue, SDNode*>(
3260 unsigned RegNo = MatcherTable[MatcherIndex++];
3261 RegNo |= MatcherTable[MatcherIndex++] << 8;
3262 RecordedNodes.
push_back(std::pair<SDValue, SDNode*>(
3269 unsigned RecNo = MatcherTable[MatcherIndex++];
3270 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitConvertToTarget");
3271 SDValue Imm = RecordedNodes[RecNo].first;
3274 const ConstantInt *Val=cast<ConstantSDNode>(Imm)->getConstantIntValue();
3276 Imm.getValueType());
3278 const ConstantFP *Val=cast<ConstantFPSDNode>(Imm)->getConstantFPValue();
3280 Imm.getValueType());
3283 RecordedNodes.
push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3292 "EmitMergeInputChains should be the first chain producing node");
3294 "Should only have one EmitMergeInputChains per match");
3298 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitMergeInputChains");
3299 ChainNodesMatched.
push_back(RecordedNodes[RecNo].first.getNode());
3303 if (ChainNodesMatched.
back() != NodeToMatch &&
3304 !RecordedNodes[RecNo].first.hasOneUse()) {
3305 ChainNodesMatched.
clear();
3319 "EmitMergeInputChains should be the first chain producing node");
3326 unsigned NumChains = MatcherTable[MatcherIndex++];
3327 assert(NumChains != 0 &&
"Can't TF zero chains");
3330 "Should only have one EmitMergeInputChains per match");
3333 for (
unsigned i = 0;
i != NumChains; ++
i) {
3334 unsigned RecNo = MatcherTable[MatcherIndex++];
3335 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitMergeInputChains");
3336 ChainNodesMatched.
push_back(RecordedNodes[RecNo].first.getNode());
3340 if (ChainNodesMatched.
back() != NodeToMatch &&
3341 !RecordedNodes[RecNo].first.hasOneUse()) {
3342 ChainNodesMatched.
clear();
3348 if (ChainNodesMatched.
empty())
3361 unsigned RecNo = MatcherTable[MatcherIndex++];
3362 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitCopyToReg");
3363 unsigned DestPhysReg = MatcherTable[MatcherIndex++];
3369 DestPhysReg, RecordedNodes[RecNo].first,
3372 InputGlue = InputChain.
getValue(1);
3377 unsigned XFormNo = MatcherTable[MatcherIndex++];
3378 unsigned RecNo = MatcherTable[MatcherIndex++];
3379 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitNodeXForm");
3381 RecordedNodes.
push_back(std::pair<SDValue,SDNode*>(Res,
nullptr));
3388 uint16_t TargetOpc = MatcherTable[MatcherIndex++];
3389 TargetOpc |= (
unsigned short)MatcherTable[MatcherIndex++] << 8;
3390 unsigned EmitNodeInfo = MatcherTable[MatcherIndex++];
3400 NumVTs = MatcherTable[MatcherIndex++];
3402 for (
unsigned i = 0;
i != NumVTs; ++
i) {
3410 if (EmitNodeInfo & OPFL_Chain)
3412 if (EmitNodeInfo & OPFL_GlueOutput)
3418 if (VTs.
size() == 1)
3420 else if (VTs.
size() == 2)
3426 unsigned NumOps = MatcherTable[MatcherIndex++];
3428 for (
unsigned i = 0;
i != NumOps; ++
i) {
3429 unsigned RecNo = MatcherTable[MatcherIndex++];
3431 RecNo =
GetVBR(RecNo, MatcherTable, MatcherIndex);
3433 assert(RecNo < RecordedNodes.
size() &&
"Invalid EmitNode");
3434 Ops.
push_back(RecordedNodes[RecNo].first);
3441 FirstOpToCopy += (EmitNodeInfo &
OPFL_Chain) ? 1 : 0;
3443 "Invalid variadic node");
3455 if (EmitNodeInfo & OPFL_Chain)
3464 if (!IsMorphNodeTo) {
3471 for (
unsigned i = 0, e = VTs.
size();
i != e; ++
i) {
3479 "NodeToMatch was removed partway through selection");
3482 auto &Chain = ChainNodesMatched;
3484 "Chain node replaced during MorphNode");
3485 Chain.erase(
std::remove(Chain.begin(), Chain.end(),
N), Chain.end());
3487 Res = MorphNode(NodeToMatch, TargetOpc, VTList, Ops, EmitNodeInfo);
3492 if (EmitNodeInfo & OPFL_GlueOutput) {
3494 if (EmitNodeInfo & OPFL_Chain)
3496 }
else if (EmitNodeInfo & OPFL_Chain)
3509 bool mayLoad = MCID.
mayLoad();
3512 unsigned NumMemRefs = 0;
3514 MatchedMemRefs.
begin(), E = MatchedMemRefs.
end();
I !=
E; ++
I) {
3515 if ((*I)->isLoad()) {
3518 }
else if ((*I)->isStore()) {
3531 MatchedMemRefs.
begin(), E = MatchedMemRefs.
end();
I !=
E; ++
I) {
3532 if ((*I)->isLoad()) {
3535 }
else if ((*I)->isStore()) {
3543 cast<MachineSDNode>(Res)
3544 ->setMemRefs(MemRefs, MemRefs + NumMemRefs);
3548 << (IsMorphNodeTo ?
"Morphed" :
"Created")
3552 if (IsMorphNodeTo) {
3554 UpdateChains(Res, InputChain, ChainNodesMatched,
true);
3564 unsigned NumResults = MatcherTable[MatcherIndex++];
3566 for (
unsigned i = 0;
i != NumResults; ++
i) {
3567 unsigned ResSlot = MatcherTable[MatcherIndex++];
3569 ResSlot =
GetVBR(ResSlot, MatcherTable, MatcherIndex);
3571 assert(ResSlot < RecordedNodes.
size() &&
"Invalid CompleteMatch");
3572 SDValue Res = RecordedNodes[ResSlot].first;
3574 assert(i < NodeToMatch->getNumValues() &&
3577 "Invalid number of results to complete!");
3582 Res.getValueSizeInBits()) &&
3583 "invalid replacement");
3588 UpdateChains(NodeToMatch, InputChain, ChainNodesMatched,
false);
3601 "Didn't replace all uses of the node?");
3611 DEBUG(
dbgs() <<
" Match failed at index " << CurrentOpcodeIndex <<
"\n");
3612 ++NumDAGIselRetries;
3614 if (MatchScopes.
empty()) {
3615 CannotYetSelect(NodeToMatch);
3621 MatchScope &LastScope = MatchScopes.
back();
3622 RecordedNodes.
resize(LastScope.NumRecordedNodes);
3624 NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
3625 N = NodeStack.back();
3627 if (LastScope.NumMatchedMemRefs != MatchedMemRefs.
size())
3628 MatchedMemRefs.
resize(LastScope.NumMatchedMemRefs);
3629 MatcherIndex = LastScope.FailIndex;
3631 DEBUG(
dbgs() <<
" Continuing at " << MatcherIndex <<
"\n");
3633 InputChain = LastScope.InputChain;
3634 InputGlue = LastScope.InputGlue;
3635 if (!LastScope.HasChainNodesMatched)
3636 ChainNodesMatched.
clear();
3641 unsigned NumToSkip = MatcherTable[MatcherIndex++];
3642 if (NumToSkip & 128)
3643 NumToSkip =
GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3647 if (NumToSkip != 0) {
3648 LastScope.FailIndex = MatcherIndex+NumToSkip;
3659 void SelectionDAGISel::CannotYetSelect(
SDNode *N) {
3662 Msg <<
"Cannot select: ";
3668 Msg <<
"\nIn function: " <<
MF->
getName();
3672 cast<ConstantSDNode>(N->
getOperand(HasInputChain))->getZExtValue();
3676 Msg <<
"target intrinsic %" <<
TII->
getName(iid);
3678 Msg <<
"unknown intrinsic #" << iid;
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
bool use_empty() const
Return true if there are no uses of this node.
std::vector< BitTestBlock > BitTestCases
BitTestCases - Vector of BitTestBlock structures used to communicate SwitchInst code generation infor...
void push_back(const T &Elt)
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
SelectionDAGBuilder * SDB
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
A parsed version of the target data layout string in and methods for querying it. ...
llvm::DenseMap< std::pair< const MachineBasicBlock *, const Value * >, unsigned > SwiftErrorVRegUpwardsUse
A list of upward exposed vreg uses that need to be satisfied by either a copy def or a phi node at th...
mop_iterator operands_end()
SDValue getValue(unsigned R) const
bool hasPostISelHook(QueryType Type=IgnoreBundle) const
Return true if this instruction requires adjustment after instruction selection by calling a target h...
static SDNode * findGlueUse(SDNode *N)
findGlueUse - Return use of MVT::Glue value produced by the specified SDNode.
void EmitLiveInCopies(MachineBasicBlock *EntryMBB, const TargetRegisterInfo &TRI, const TargetInstrInfo &TII)
EmitLiveInCopies - Emit copies to initialize livein virtual registers into the given entry block...
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
virtual bool enableMachineScheduler() const
True if the subtarget should run MachineScheduler after aggressive coalescing.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
static int getNumFixedFromVariadicInfo(unsigned Flags)
getNumFixedFromVariadicInfo - Transform an EmitNode flags word into the number of fixed arity values ...
bool LegalizeTypes()
This transforms the SelectionDAG into a SelectionDAG that only uses types natively supported by the t...
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
virtual bool hasCopyImplyingStackAdjustment(MachineFunction *MF) const
Return true if the MachineFunction contains a COPY which would imply HasCopyImplyingStackAdjustment.
DELETED_NODE - This is an illegal value that is used to catch errors.
STATISTIC(NumFunctions,"Total number of functions")
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR...
static unsigned virtReg2Index(unsigned Reg)
Convert a virtual register number to a 0-based index.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
livein_iterator livein_end() const
MCSymbol * addLandingPad(MachineBasicBlock *LandingPad)
Add a new panding pad. Returns the label ID for the landing pad entry.
bool hasProperty(Property P) const
static cl::opt< bool > ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected"))
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
unsigned getCatchPadExceptionPointerVReg(const Value *CPI, const TargetRegisterClass *RC)
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
static void propagateSwiftErrorVRegs(FunctionLoweringInfo *FuncInfo)
Propagate swifterror values through the machine function CFG.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
virtual void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const
Insert explicit copies in entry and exit blocks.
unsigned createVirtualRegister(const TargetRegisterClass *RegClass)
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)
Remove path.
ScheduleDAGSDNodes *(* FunctionPassCtor)(SelectionDAGISel *, CodeGenOpt::Level)
bool isReturn() const
Return true if the instruction is a return.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
bool mayStore() const
Return true if this instruction could possibly modify memory.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
Describe properties that are true of each instruction in the target description file.
void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &)
Clients of various APIs that cause global effects on the DAG can optionally implement this interface...
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in the KnownZero/KnownO...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel)
CheckPatternPredicate - Implements OP_CheckPatternPredicate.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
void visitJumpTableHeader(JumpTable &JT, JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
static const MCPhysReg VRegs[32]
bool isTokenTy() const
Return true if this is 'token'.
unsigned getNumOperands() const
Return the number of values used by this operation.
bool isEHPad() const
Return true if this basic block is an exception handling block.
unsigned getNumOperands() const
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned Num) const
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
RegisterPassParser class - Handle the addition of new machine passes.
void setNodeId(int Id)
Set unique node id.
static MachineBasicBlock::iterator FindSplitPointForStackProtector(MachineBasicBlock *BB)
Find the split point at which to splice the end of BB into its success stack protector check machine ...
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
virtual unsigned getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
void viewGraph(const std::string &Title)
Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL, unsigned ChildNo)
DebugLoc getCurDebugLoc() const
EntryToken - This is the marker used to indicate the start of a region.
void setCurrentSwiftErrorVReg(const MachineBasicBlock *MBB, const Value *, unsigned)
Set the swifterror virtual register in the SwiftErrorVRegDefMap for this basic block.
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
const TargetLibraryInfo * LibInfo
void set(const Function &Fn, MachineFunction &MF, SelectionDAG *DAG)
set - Initialize this FunctionLoweringInfo with the given Function and its associated MachineFunction...
static void setupSwiftErrorVals(const Function &Fn, const TargetLowering *TLI, FunctionLoweringInfo *FuncInfo)
Set up SwiftErrorVals by going through the function.
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
StringRef getName() const
Return a constant reference to the value's name.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
iterator begin()
Instruction iterator methods.
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
static bool isFoldedOrDeadInstruction(const Instruction *I, FunctionLoweringInfo *FuncInfo)
isFoldedOrDeadInstruction - Return true if the specified instruction is side-effect free and is eithe...
StackProtectorDescriptor SPDescriptor
A StackProtectorDescriptor structure used to communicate stack protector information in between Selec...
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool isVector() const
isVector - Return true if this is a vector value type.
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
SDValue getRoot()
getRoot - Return the current virtual root of the Selection DAG, flushing any PendingLoad items...
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block...
AnalysisUsage & addRequired()
void setLastLocalValue(MachineInstr *I)
Update the position of the last instruction emitted for materializing constants for use in the curren...
A description of a memory reference used in the backend.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
static bool findNonImmUse(SDNode *Use, SDNode *Def, SDNode *ImmedUse, SDNode *Root, SmallPtrSetImpl< SDNode * > &Visited, bool IgnoreChains)
findNonImmUse - Return true if "Use" is a non-immediate use of "Def".
bool CheckAndMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckAndMask - The isel is trying to match something like (and X, 255).
void visitSwitchCase(CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
struct fuzzer::@269 Flags
Option class for critical edge splitting.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOpt::Level OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
This class is basically a combination of TimeRegion and Timer.
void AddLiveOutRegInfo(unsigned Reg, unsigned NumSignBits, const APInt &KnownZero, const APInt &KnownOne)
AddLiveOutRegInfo - Adds LiveOutInfo for a register.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
A Use represents the edge between a Value definition and its users.
static SDValue HandleMergeInputChains(SmallVectorImpl< SDNode * > &ChainNodesMatched, SelectionDAG *CurDAG)
HandleMergeInputChains - This implements the OPC_EmitMergeInputChains operation for when the pattern ...
const Value * SwiftErrorArg
The swifterror argument of the current function.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
void visitJumpTable(JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
DenseMap< const Value *, unsigned > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
use_instr_iterator use_instr_begin(unsigned RegNo) const
const TargetLowering * TLI
#define LLVM_ATTRIBUTE_ALWAYS_INLINE
LLVM_ATTRIBUTE_ALWAYS_INLINE - On compilers where we have a directive to do so, mark a method "always...
bool isCall() const
Return true if the instruction is a call.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
bool isReg() const
isReg - Tests if this is a MO_Register operand.
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
CopyToReg - This node has three operands: a chain, a register number to set to this value...
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
static MachinePassRegistry Registry
RegisterScheduler class - Track the registration of instruction schedulers.
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
const MachineFunctionProperties & getProperties() const
Get the function properties.
void freezeReservedRegs(const MachineFunction &)
freezeReservedRegs - Called by the register allocator to freeze the set of reserved registers before ...
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Reg
All possible values of the reg field in the ModR/M byte.
virtual void Select(SDNode *N)=0
Main hook for targets to transform nodes into machine nodes.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
An analysis pass which caches information about the entire Module.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N)
virtual unsigned getFrameRegister(const MachineFunction &MF) const =0
Debug information queries.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
INLINEASM - Represents an inline asm block.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
LLVM_NODISCARD bool empty() const
unsigned DAGSize
DAGSize - Size of DAG being instruction selected.
void initializeAAResultsWrapperPassPass(PassRegistry &)
static cl::opt< bool > EnableFastISelVerbose("fast-isel-verbose", cl::Hidden, cl::desc("Enable verbose messages in the \"fast\" ""instruction selector"))
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
defusechain_iterator - This class provides iterator support for machine operands in the function that...
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
const MachineBasicBlock & front() const
static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI)
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
A constant value that is initialized with an expression using other constant values.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
ScheduleDAGSDNodes * createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createHybridListDAGScheduler - This creates a bottom up register pressure aware list scheduler that m...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
virtual void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag...
void setHasCopyImplyingStackAdjustment(bool B)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
const APInt & getAPIntValue() const
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(std::begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
MachinePassRegistry - Track the registration of machine passes.
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification, or lowering of the constant.
~SelectionDAGISel() override
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
static LLVM_ATTRIBUTE_ALWAYS_INLINE uint64_t GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx)
GetVBR - decode a vbr encoding whose top bit is set.
ScheduleDAGSDNodes * createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up list scheduler that schedules nodes in source c...
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
void init(MachineFunction &mf)
Prepare this SelectionDAG to process code in the given MachineFunction.
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
const DataLayout & getDataLayout() const
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
Legacy analysis pass which computes BranchProbabilityInfo.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
UNDEF - An undefined node.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
unsigned getNumIncomingValues() const
Return the number of incoming edges.
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
const MachineBasicBlock * getParent() const
TargetInstrInfo - Interface to description of machine instruction set.
This corresponds to the llvm.lifetime.
MachineRegisterInfo * RegInfo
void clear()
clear - Clear out all the function-specific state.
virtual void initializeSplitCSR(MachineBasicBlock *Entry) const
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
bool isDebugValue() const
bool isImplicitDef() const
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
SDNode * getNode() const
get the SDNode which holds the desired result
static cl::opt< RegisterScheduler::FunctionPassCtor, false, RegisterPassParser< RegisterScheduler > > ISHeuristic("pre-RA-sched", cl::init(&createDefaultScheduler), cl::Hidden, cl::desc("Instruction schedulers available (before register"" allocation):"))
ISHeuristic command line option for instruction schedulers.
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
BasicBlock * SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions())
If this edge is a critical edge, insert a new node to split the critical edge.
CriticalEdgeSplittingOptions & setMergeIdenticalEdges()
initializer< Ty > init(const Ty &Val)
virtual FastISel * createFastISel(FunctionLoweringInfo &, const TargetLibraryInfo *) const
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry &)
llvm::DenseMap< std::pair< const MachineBasicBlock *, const Value * >, unsigned > SwiftErrorVRegDefMap
A map from swifterror value in a basic block to the virtual register it is currently represented by...
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
CodeGenOpt::Level OptLevel
void addLiveIn(MCPhysReg PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
static cl::opt< int > EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction selection ""fails to lower an instruction: 0 disable the abort, 1 will ""abort but for args, calls and terminators, 2 will also ""abort for argument lowering, and 3 will never fallback ""to SelectionDAG."))
Subclasses of this class are all able to terminate a basic block.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
unsigned const MachineRegisterInfo * MRI
virtual bool CheckNodePredicate(SDNode *N, unsigned PredNo) const
CheckNodePredicate - This function is generated by tblgen in the target.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
ScheduleDAGSDNodes * createVLIWDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createVLIWDAGScheduler - Scheduler for VLIW targets.
bool HasTailCall
HasTailCall - This is set to true if a call in the current block has been translated as a tail call...
bool getO0WantsFastISel()
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
ScheduleDAGSDNodes * createDefaultScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDefaultScheduler - This creates an instruction scheduler appropriate for the target...
MVT - Machine Value Type.
LLVM Basic Block Representation.
const SDValue & getOperand(unsigned i) const
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
bool canTrap() const
Return true if evaluation of this constant could trap.
ScheduleDAGSDNodes * createBURRListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up register usage reduction list scheduler...
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineInstrBuilder & UseMI
void Combine(CombineLevel Level, AliasAnalysis &AA, CodeGenOpt::Level OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together, or eliminating superfluous nodes.
virtual bool supportSplitCSR(MachineFunction *MF) const
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
This is an important base class in LLVM.
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
bool isIndirectDebugValue() const
A DBG_VALUE is indirect iff the first operand is a register and the second operand is an immediate...
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
const MachineOperand & getOperand(unsigned i) const
bool succ_empty(const BasicBlock *BB)
static ChainResult WalkChainUsers(const SDNode *ChainedNode, SmallVectorImpl< SDNode * > &ChainedNodesInPattern, DenseMap< const SDNode *, ChainResult > &TokenFactorResult, SmallVectorImpl< SDNode * > &InteriorChainedNodes)
WalkChainUsers - Walk down the users of the specified chained node that is part of the pattern we're ...
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
ConstantFP - Floating Point Values [float, double].
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
Represent the analysis usage information of a pass.
This class provides iterator support for SDUse operands that use a specific SDNode.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We're checking to see if we can fold LI into FoldInst.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, unsigned ChildNo)
static unsigned getMemoryConstraintID(unsigned Flag)
void printrFull(raw_ostream &O, const SelectionDAG *G=nullptr) const
Print a SelectionDAG node and all children down to the leaves.
std::vector< NodeRef >::reverse_iterator rpo_iterator
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
virtual unsigned getRegisterByName(const char *RegName, EVT VT, SelectionDAG &DAG) const
Return the register ID of the name passed in.
static const unsigned End
static void collectFailStats(const Instruction *I)
unsigned getOpcode() const
static cl::opt< bool > UseMBPI("use-mbpi", cl::desc("use Machine Branch Probability Info"), cl::init(true), cl::Hidden)
Interval::pred_iterator pred_end(Interval *I)
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
void setHasInlineAsm(bool B)
Set a flag that indicates that the function contains inline assembly.
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection...
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
static void createSwiftErrorEntriesInEntryBlock(FunctionLoweringInfo *FuncInfo, const TargetLowering *TLI, const TargetInstrInfo *TII, const BasicBlock *LLVMBB, SelectionDAGBuilder *SDB)
self_iterator getIterator()
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
static cl::opt< bool > ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the second ""dag combine pass"))
iterator_range< pred_iterator > predecessors()
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
static unsigned getFlagWordForMem(unsigned InputFlag, unsigned Constraint)
Augment an existing flag word returned by getFlagWord with the constraint code for a memory constrain...
const DIExpression * getDebugExpression() const
Return the complex address expression referenced by this DBG_VALUE instruction.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
unsigned ExceptionPointerVirtReg
If the current MBB is a landing pad, the exception pointer and exception selector registers are copie...
SmallPtrSet< const BasicBlock *, 4 > VisitedBBs
VisitedBBs - The set of basic blocks visited thus far by instruction selection.
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
static bool isMemKind(unsigned Flag)
EVT - Extended Value Type.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
void ComputePHILiveOutRegInfo(const PHINode *)
ComputePHILiveOutRegInfo - Compute LiveOutInfo for a PHI's destination register based on the LiveOutI...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
HANDLENODE node - Used as a handle for various purposes.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
MachineBasicBlock * MBB
MBB - The current block.
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
virtual bool enableMachineSchedDefaultSched() const
True if the machine scheduler should disable the TLI preference for preRA scheduling with the source ...
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
void dump() const
Support for debugging, callable in GDB: V->dump()
TargetIntrinsicInfo - Interface to description of machine instruction set.
static cl::opt< bool > ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the first ""dag combine pass"))
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
virtual Value * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
bool hasCalls() const
Return true if the current function has any function calls.
bool SplitCSR
True if part of the CSRs will be handled via explicit copies.
TokenFactor - This node takes multiple tokens as input and produces a single token result...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
const MDOperand & getOperand(unsigned I) const
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
This is the shared class of boolean and integer constants.
ScheduleDAGSDNodes * createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createILPListDAGScheduler - This creates a bottom up register pressure aware list scheduler that trie...
void dump() const
Dump this node, for debugging.
allnodes_const_iterator allnodes_begin() const
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
DenseMap< unsigned, unsigned > RegFixups
RegFixups - Registers which need to be replaced after isel is done.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDNode * SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type...
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
SmallVector< MachineInstr *, 8 > ArgDbgValues
ArgDbgValues - A list of DBG_VALUE instructions created during isel for function arguments that are i...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
void clear()
clear - Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuil...
virtual const TargetLowering * getTargetLowering() const
void setFastISel(bool Enable)
SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL=CodeGenOpt::Default)
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
void visit(const Instruction &I)
LLVM_NODISCARD T pop_back_val()
bool mayLoad() const
Return true if this instruction could possibly read memory.
livein_iterator livein_begin() const
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel, SDNode *N)
CheckNodePredicate - Implements OP_CheckNodePredicate.
Represents one node in the SelectionDAG.
const BasicBlock & getEntryBlock() const
virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N, unsigned PatternNo, SmallVectorImpl< std::pair< SDValue, SDNode * > > &Result)
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
UpdateSplitBlock - When an MBB was split during scheduling, update the references that need to refer ...
static cl::opt< bool > ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize types"))
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
unsigned CreateRegs(Type *Ty)
CreateRegs - Allocate the appropriate number of virtual registers of the correctly promoted or expand...
void startNewBlock()
Set the current block to which generated machine instructions will be appended, and clear the local C...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static cl::opt< bool > ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize"))
void setCallSiteLandingPad(MCSymbol *Sym, ArrayRef< unsigned > Sites)
Map the landing pad's EH symbol to the call site indexes.
StringRef getName() const
Return the name of the corresponding LLVM basic block, or "(null)".
Class for arbitrary precision integers.
static unsigned IsPredicateKnownToFail(const unsigned char *Table, unsigned Index, SDValue N, bool &Result, const SelectionDAGISel &SDISel, SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
IsPredicateKnownToFail - If we know how and can do so without pushing a scope, evaluate the current n...
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
void visitBitTestCase(BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, unsigned Reg, BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
BranchProbabilityInfo * BPI
machine Machine Instruction Scheduler
int64_t getSExtValue() const
op_iterator op_begin() const
This file defines the FastISel class.
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
static use_iterator use_end()
iterator_range< user_iterator > users()
std::vector< JumpTableBlock > JTCases
JTCases - Vector of JumpTable structures used to communicate SwitchInst code generation information...
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void replaceRegWith(unsigned FromReg, unsigned ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes, unsigned ChildNo)
CheckChildSame - Implements OP_CheckChildXSame.
iterator_range< value_op_iterator > op_values() const
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
TargetSubtargetInfo - Generic base class for all target subtargets.
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
Representation of each machine instruction.
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Represents a use of a SDNode.
unsigned ExceptionSelectorVirtReg
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ)
Search for the specified successor of basic block BB and return its position in the terminator instru...
Bitwise operators - logical and, logical or, logical xor.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
virtual void viewGraph(const Twine &Name, const Twine &Title)
viewGraph - Pop up a GraphViz/gv window with the ScheduleDAG rendered using 'dot'.
static bool MIIsInTerminatorSequence(const MachineInstr &MI)
Given that the input MI is before a partial terminator sequence TSeq, return true if M + TSeq also a ...
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
void initializeGCModuleInfoPass(PassRegistry &)
static cl::opt< bool > ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed"))
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
bool isLandingPad() const
Return true if this basic block is a landing pad.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
bool hasOneUse() const
Return true if there is exactly one user of this value.
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
bool callsFunctionThatReturnsTwice() const
callsFunctionThatReturnsTwice - Return true if the function has a call to setjmp or other function th...
void resetTargetOptions(const Function &F) const
Reset the target options based on the function's attributes.
void clearKillFlags(unsigned Reg) const
clearKillFlags - Iterate over all the uses of the given register and clear the kill flag from the Mac...
iterator find(const KeyT &Val)
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
op_iterator op_end() const
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
void Run(SelectionDAG *dag, MachineBasicBlock *bb)
Run - perform scheduling.
OptLevelChanger(SelectionDAGISel &ISel, CodeGenOpt::Level NewOptLevel)
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
void setOptLevel(CodeGenOpt::Level Level)
Overrides the optimization level.
bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckOrMask - The isel is trying to match something like (or X, 255).
int getNodeId() const
Return the unique node id.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
EVT getValueType() const
Return the ValueType of the referenced return value.
MachineInstr * getVRegDef(unsigned Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode...
const DILocalVariable * getDebugVariable() const
Return the debug variable referenced by this DBG_VALUE instruction.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
std::vector< std::pair< unsigned, unsigned > >::const_iterator livein_iterator
unsigned getReg() const
getReg - Returns the register number.
const MDNode * getMD() const
virtual bool CheckPatternPredicate(unsigned PredNo) const
CheckPatternPredicate - This function is generated by tblgen in the target.
This class is used to form a handle around another node that is persistent and is updated across invo...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A raw_ostream that writes to an std::string.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void InvalidatePHILiveOutRegInfo(const PHINode *PN)
InvalidatePHILiveOutRegInfo - Invalidates a PHI's LiveOutInfo, to be called when a block is visited b...
virtual const TargetInstrInfo * getInstrInfo() const
Module * getParent()
Get the module that this global value is contained inside of...
SDValue getRegister(unsigned Reg, EVT VT)
mop_iterator operands_begin()
void init(GCFunctionInfo *gfi, AliasAnalysis &aa, const TargetLibraryInfo *li)
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void SelectInlineAsmMemoryOperands(std::vector< SDValue > &Ops, const SDLoc &DL)
SelectInlineAsmMemoryOperands - Calls to this are automatically generated by tblgen.
virtual MachineBasicBlock * EmitSchedule(MachineBasicBlock::iterator &InsertPos)
EmitSchedule - Insert MachineInstrs into the MachineBasicBlock according to the order specified in Se...
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
static use_instr_iterator use_instr_end()
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
CheckSame - Implements OP_CheckSame.
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
static cl::opt< bool > ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed"))
std::vector< CaseBlock > SwitchCases
SwitchCases - Vector of CaseBlock structures used to communicate SwitchInst code generation informati...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.Val alone...
Primary interface to the complete machine description for the target machine.
DenseMap< const BasicBlock *, MachineBasicBlock * > MBBMap
MBBMap - A mapping from LLVM basic blocks to their machine code entry.
SDValue getControlRoot()
getControlRoot - Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports items.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
StringRef - Represent a constant reference to a string, i.e.
void clearDanglingDebugInfo()
clearDanglingDebugInfo - Clear the dangling debug information map.
SwiftErrorValues SwiftErrorVals
A function can only have a single swifterror argument.
const TargetLowering * TLI
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
static cl::opt< bool > EnableFastISelVerbose2("fast-isel-verbose2", cl::Hidden, cl::desc("Enable extra verbose messages in the \"fast\" ""instruction selector"))
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
LPadToCallSiteMap - Map a landing pad to the call site indexes.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
bool TimePassesIsEnabled
If the user specifies the -time-passes argument on an LLVM tool command line then the value of this b...
SDNode * getUser()
This returns the SDNode that contains this Use.
auto find_if(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range))
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly...
virtual RegisterScheduler::FunctionPassCtor getDAGScheduler(CodeGenOpt::Level) const
Target can subclass this hook to select a different DAG scheduler.
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
static cl::opt< std::string > FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, cl::desc("Only display the basic block whose name ""matches this for all view-*-dags options"))
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const TargetInstrInfo * TII
virtual unsigned getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num)
allocateMemRefsArray - Allocate an array to hold MachineMemOperand pointers.
void visitBitTestHeader(BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests" ...
static void SplitCriticalSideEffectEdges(Function &Fn)
SplitCriticalSideEffectEdges - Look for critical edges with a PHI value that may trap on it...
bool isExportedInst(const Value *V)
isExportedInst - Return true if the specified value is an instruction exported from its block...
unsigned getResNo() const
Convenience function for get().getResNo().
FunctionLoweringInfo * FuncInfo
static cl::opt< bool > ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the post legalize types"" dag combine pass"))
static RegisterScheduler defaultListDAGScheduler("default","Best scheduler for the target", createDefaultScheduler)
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
A wrapper class for inspecting calls to intrinsic functions.
This file describes how to lower LLVM code to machine code.
virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo)
bool isVoidTy() const
Return true if this is 'void'.
an instruction to allocate memory on the stack
bool use_empty(unsigned RegNo) const
use_empty - Return true if there are no instructions using the specified register.
This class is used by SelectionDAGISel to temporarily override the optimization level on a per-functi...
void SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, unsigned TableSize)
unsigned getOrCreateSwiftErrorVReg(const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register in SwiftErrorVRegDefMap for this basic block...
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.