36 #define DEBUG_TYPE "misched"
40 cl::desc(
"Force top-down list scheduling"));
42 cl::desc(
"Force bottom-up list scheduling"));
45 cl::desc(
"Print critical path length to stdout"));
50 cl::desc(
"Pop up a window to show MISched dags after they are processed"));
56 cl::desc(
"Only schedule this function"));
58 cl::desc(
"Only schedule this MBB#"));
77 cl::desc(
"Verify machine instrs before and after machine scheduling"));
83 void MachineSchedStrategy::anchor() {}
84 void ScheduleDAGMutation::anchor() {}
91 MF(nullptr), MLI(nullptr), MDT(nullptr), PassConfig(nullptr), AA(nullptr), LIS(nullptr) {
113 class MachineScheduler :
public MachineSchedulerBase {
128 class PostMachineScheduler :
public MachineSchedulerBase {
130 PostMachineScheduler();
148 "Machine Instruction Scheduler",
false,
false)
155 MachineScheduler::MachineScheduler()
156 : MachineSchedulerBase(
ID) {
160 void MachineScheduler::getAnalysisUsage(
AnalysisUsage &AU)
const {
178 "PostRA Machine Instruction Scheduler",
false,
false)
180 PostMachineScheduler::PostMachineScheduler()
181 : MachineSchedulerBase(
ID) {
185 void PostMachineScheduler::getAnalysisUsage(
AnalysisUsage &AU)
const {
206 cl::desc(
"Machine instruction scheduler to use"));
214 cl::desc(
"Enable the machine instruction scheduling pass."),
cl::init(
true),
226 assert(I != Beg &&
"reached the top of the region, cannot decrement");
228 if (!I->isDebugValue())
247 for(; I != End; ++
I) {
248 if (!I->isDebugValue())
262 const_cast<MachineInstr*>(
322 MLI = &getAnalysis<MachineLoopInfo>();
323 MDT = &getAnalysis<MachineDominatorTree>();
324 PassConfig = &getAnalysis<TargetPassConfig>();
325 AA = &getAnalysis<AliasAnalysis>();
327 LIS = &getAnalysis<LiveIntervals>();
331 MF->verify(
this,
"Before machine scheduling.");
333 RegClassInfo->runOnMachineFunction(*MF);
337 std::unique_ptr<ScheduleDAGInstrs>
Scheduler(createMachineScheduler());
338 scheduleRegions(*Scheduler);
342 MF->verify(
this,
"After machine scheduling.");
351 DEBUG(
dbgs() <<
"Subtarget disables post-MI-sched.\n");
358 PassConfig = &getAnalysis<TargetPassConfig>();
361 MF->
verify(
this,
"Before post machine scheduling.");
365 std::unique_ptr<ScheduleDAGInstrs>
Scheduler(createPostMachineScheduler());
366 scheduleRegions(*Scheduler);
369 MF->verify(
this,
"After post machine scheduling.");
394 bool IsPostRA = Scheduler.
isPostRA();
401 MBB != MBBEnd; ++MBB) {
426 unsigned RemainingInstrs = std::distance(MBB->begin(), MBB->end());
428 RegionEnd != MBB->begin(); RegionEnd = Scheduler.
begin()) {
431 if (RegionEnd != MBB->end() ||
440 unsigned NumRegionInstrs = 0;
442 for(;I != MBB->begin(); --
I, --RemainingInstrs) {
445 if (!I->isDebugValue())
450 Scheduler.
enterRegion(MBB, I, RegionEnd, NumRegionInstrs);
453 if (I == RegionEnd || I == std::prev(RegionEnd)) {
460 <<
"MI Scheduling **********\n");
462 <<
":BB#" << MBB->getNumber() <<
" " << MBB->getName()
463 <<
"\n From: " << *I <<
" To: ";
464 if (RegionEnd != MBB->end())
dbgs() << *RegionEnd;
465 else dbgs() <<
"End";
466 dbgs() <<
" RegionInstrs: " << NumRegionInstrs
467 <<
" Remaining: " << RemainingInstrs <<
"\n");
469 errs() << MF->getName();
470 errs() <<
":BB# " << MBB->getNumber();
471 errs() <<
" " << MBB->getName() <<
" \n";
483 RegionEnd = Scheduler.
begin();
485 assert(RemainingInstrs == 0 &&
"Instruction count mismatch!");
502 dbgs() << Name <<
": ";
503 for (
unsigned i = 0, e = Queue.size(); i < e; ++i)
504 dbgs() << Queue[i]->NodeNum <<
" ";
550 dbgs() <<
"*** Scheduling failed! ***\n";
552 dbgs() <<
" has been released too many times!\n";
589 dbgs() <<
"*** Scheduling failed! ***\n";
591 dbgs() <<
" has been released too many times!\n";
620 unsigned regioninstrs)
624 SchedImpl->initPolicy(begin, end, regioninstrs);
640 LIS->handleMove(MI,
true);
677 DEBUG(
for (
unsigned su = 0, e =
SUnits.size(); su != e; ++su)
678 SUnits[su].dumpAll(
this));
684 bool IsTopNode =
false;
686 assert(!SU->isScheduled &&
"Node already scheduled");
692 assert(SU->isTopReady() &&
"node still has unscheduled dependencies");
699 assert(SU->isBottomReady() &&
"node still has unscheduled dependencies");
724 unsigned BBNum =
begin()->getParent()->getNumber();
725 dbgs() <<
"*** Final schedule for BB#" << BBNum <<
" ***\n";
733 for (
unsigned i = 0, e =
Mutations.size(); i < e; ++i) {
741 for (std::vector<SUnit>::iterator
744 assert(!SU->
isBoundaryNode() &&
"Boundary node should not be in SUnits");
750 if (!I->NumPredsLeft)
753 if (!I->NumSuccsLeft)
770 I = TopRoots.
begin(), E = TopRoots.
end(); I != E; ++
I) {
776 I = BotRoots.
rbegin(), E = BotRoots.
rend(); I != E; ++
I) {
809 for (std::vector<std::pair<MachineInstr *, MachineInstr *> >::iterator
811 std::pair<MachineInstr *, MachineInstr *>
P = *std::prev(
DI);
824 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
830 dbgs() <<
"Missing SUnit\n";
851 unsigned regioninstrs)
910 for (
unsigned i = 0, e = RegionPressure.size(); i < e; ++i) {
912 if (RegionPressure[i] > Limit) {
914 <<
" Limit " << Limit
915 <<
" Actual " << RegionPressure[i] <<
"\n");
928 const std::vector<unsigned> &NewMaxPressure) {
935 unsigned ID = I->getPSet();
940 && NewMaxPressure[ID] <= INT16_MAX)
944 if (NewMaxPressure[ID] >= Limit - 2) {
946 << NewMaxPressure[
ID]
947 << ((NewMaxPressure[
ID] > Limit) ?
" > " :
" <= ") << Limit
956 for (
unsigned LUIdx = 0, LUEnd = LiveUses.
size(); LUIdx != LUEnd; ++LUIdx) {
958 unsigned Reg = LiveUses[LUIdx];
978 assert(VNI &&
"No live value at use.");
1020 DEBUG(
for (
unsigned su = 0, e =
SUnits.size(); su != e; ++su)
1021 SUnits[su].dumpAll(
this));
1032 bool IsTopNode =
false;
1034 assert(!SU->isScheduled &&
"Node already scheduled");
1059 unsigned BBNum =
begin()->getParent()->getNumber();
1060 dbgs() <<
"*** Final schedule for BB#" << BBNum <<
" ***\n";
1131 unsigned MaxCyclicLatency = 0;
1149 unsigned LiveOutHeight = DefSU->
getHeight();
1159 LI.
Query(LIS->getInstructionIndex(UI->SU->getInstr()));
1166 unsigned CyclicLatency = 0;
1167 if (LiveOutDepth > UI->SU->getDepth())
1168 CyclicLatency = LiveOutDepth - UI->SU->getDepth();
1170 unsigned LiveInHeight = UI->SU->getHeight() + DefSU->
Latency;
1171 if (LiveInHeight > LiveOutHeight) {
1172 if (LiveInHeight - LiveOutHeight < CyclicLatency)
1173 CyclicLatency = LiveInHeight - LiveOutHeight;
1179 << UI->SU->NodeNum <<
") = " << CyclicLatency <<
"c\n");
1180 if (CyclicLatency > MaxCyclicLatency)
1181 MaxCyclicLatency = CyclicLatency;
1184 DEBUG(
dbgs() <<
"Cyclic Critical Path: " << MaxCyclicLatency <<
"c\n");
1185 return MaxCyclicLatency;
1194 assert(SU->
isTopReady() &&
"node still has unscheduled dependencies");
1210 assert(SU->
isBottomReady() &&
"node still has unscheduled dependencies");
1213 if (&*priorII == MI)
1246 LoadInfo(
SUnit *su,
unsigned reg,
unsigned ofs)
1247 : SU(su), BaseReg(reg), Offset(ofs) {}
1249 bool operator<(
const LoadInfo &RHS)
const {
1250 return std::tie(BaseReg, Offset) < std::tie(RHS.BaseReg, RHS.Offset);
1259 : TII(tii), TRI(tri) {}
1270 for (
unsigned Idx = 0, End = Loads.
size(); Idx != End; ++Idx) {
1271 SUnit *SU = Loads[Idx];
1275 LoadRecords.
push_back(LoadInfo(SU, BaseReg, Offset));
1277 if (LoadRecords.
size() < 2)
1279 std::sort(LoadRecords.
begin(), LoadRecords.
end());
1280 unsigned ClusterLength = 1;
1281 for (
unsigned Idx = 0, End = LoadRecords.
size(); Idx < (End - 1); ++Idx) {
1282 if (LoadRecords[Idx].BaseReg != LoadRecords[Idx+1].BaseReg) {
1287 SUnit *SUa = LoadRecords[Idx].SU;
1288 SUnit *SUb = LoadRecords[Idx+1].SU;
1299 SI = SUa->
Succs.begin(), SE = SUa->
Succs.end();
SI != SE; ++
SI) {
1300 if (
SI->getSUnit() == SUb)
1302 DEBUG(
dbgs() <<
" Copy Succ SU(" <<
SI->getSUnit()->NodeNum <<
")\n");
1318 for (
unsigned Idx = 0, End = DAG->
SUnits.size(); Idx != End; ++Idx) {
1322 unsigned ChainPredID = DAG->
SUnits.size();
1324 PI = SU->
Preds.begin(), PE = SU->
Preds.end(); PI != PE; ++PI) {
1326 ChainPredID = PI->getSUnit()->NodeNum;
1332 unsigned NumChains = StoreChainDependents.
size();
1333 std::pair<DenseMap<unsigned, unsigned>::iterator,
bool> Result =
1334 StoreChainIDs.
insert(std::make_pair(ChainPredID, NumChains));
1336 StoreChainDependents.
resize(NumChains + 1);
1337 StoreChainDependents[Result.first->second].
push_back(SU);
1340 for (
unsigned Idx = 0, End = StoreChainDependents.
size(); Idx != End; ++Idx)
1341 clusterNeighboringLoads(StoreChainDependents[Idx], DAG);
1368 for (
unsigned Idx = DAG->
SUnits.size(); Idx > 0;) {
1381 assert(Success &&
"No DAG nodes should be reachable from ExitSU");
1450 unsigned LocalReg = SrcReg;
1451 unsigned GlobalReg = DstReg;
1453 if (!LocalLI->
isLocal(RegionBeginIdx, RegionEndIdx)) {
1457 if (!LocalLI->
isLocal(RegionBeginIdx, RegionEndIdx))
1468 if (GlobalSegment == GlobalLI->
end())
1475 if (GlobalSegment->contains(LocalLI->
beginIndex()))
1478 if (GlobalSegment == GlobalLI->
end())
1482 if (GlobalSegment != GlobalLI->
begin()) {
1485 GlobalSegment->start)) {
1496 assert(std::prev(GlobalSegment)->start < LocalLI->beginIndex() &&
1497 "Disconnected LRG within the scheduling region.");
1514 I = LastLocalSU->
Succs.begin(), E = LastLocalSU->
Succs.end();
1516 if (I->getKind() !=
SDep::Data || I->getReg() != LocalReg)
1518 if (I->getSUnit() == GlobalSU)
1520 if (!DAG->
canAddEdge(GlobalSU, I->getSUnit()))
1531 I = GlobalSU->
Preds.begin(), E = GlobalSU->
Preds.end(); I != E; ++
I) {
1532 if (I->getKind() !=
SDep::Anti || I->getReg() != GlobalReg)
1534 if (I->getSUnit() == FirstLocalSU)
1536 if (!DAG->
canAddEdge(FirstLocalSU, I->getSUnit()))
1543 I = LocalUses.
begin(), E = LocalUses.
end(); I != E; ++
I) {
1544 DEBUG(
dbgs() <<
" Local use SU(" << (*I)->NodeNum <<
") -> SU("
1545 << GlobalSU->
NodeNum <<
")\n");
1549 I = GlobalUses.
begin(), E = GlobalUses.
end(); I != E; ++
I) {
1550 DEBUG(
dbgs() <<
" Global use SU(" << (*I)->NodeNum <<
") -> SU("
1551 << FirstLocalSU->
NodeNum <<
")\n");
1562 if (FirstPos == DAG->
end())
1568 for (
unsigned Idx = 0, End = DAG->
SUnits.size(); Idx != End; ++Idx) {
1573 constrainLocalCopy(SU, static_cast<ScheduleDAGMILive*>(DAG));
1596 CheckPending =
false;
1600 MinReadyCycle = UINT_MAX;
1601 ExpectedLatency = 0;
1602 DependentLatency = 0;
1604 MaxExecutedResCount = 0;
1606 IsResourceLimited =
false;
1607 ReservedCycles.
clear();
1612 MaxObservedStall = 0;
1615 ExecutedResCounts.
resize(1);
1616 assert(!ExecutedResCounts[0] &&
"nonzero count for bad resource");
1625 for (std::vector<SUnit>::iterator
1626 I = DAG->
SUnits.begin(), E = DAG->
SUnits.end(); I != E; ++
I) {
1633 unsigned PIdx = PI->ProcResourceIdx;
1664 if (ReadyCycle > CurrCycle)
1665 return ReadyCycle - CurrCycle;
1673 unsigned NextUnreserved = ReservedCycles[PIdx];
1679 NextUnreserved += Cycles;
1680 return NextUnreserved;
1713 if (NRCycle > CurrCycle) {
1715 MaxObservedStall = std::max(PI->Cycles, MaxObservedStall);
1719 <<
"=" << NRCycle <<
"c\n");
1730 SUnit *LateSU =
nullptr;
1731 unsigned RemLatency = 0;
1735 if (L > RemLatency) {
1742 << LateSU->
NodeNum <<
") " << RemLatency <<
"c\n");
1761 PIdx != PEnd; ++PIdx) {
1763 if (OtherCount > OtherCritCount) {
1764 OtherCritCount = OtherCount;
1765 OtherCritIdx = PIdx;
1773 return OtherCritCount;
1777 assert(SU->
getInstr() &&
"Scheduled SUnit must have instr");
1783 if (ReadyCycle > CurrCycle)
1784 MaxObservedStall = std::max(ReadyCycle - CurrCycle, MaxObservedStall);
1787 if (ReadyCycle < MinReadyCycle)
1788 MinReadyCycle = ReadyCycle;
1793 if ((!IsBuffered && ReadyCycle > CurrCycle) ||
checkHazard(SU))
1819 assert(MinReadyCycle < UINT_MAX &&
"MinReadyCycle uninitialized");
1820 if (MinReadyCycle > NextCycle)
1821 NextCycle = MinReadyCycle;
1825 CurrMOps = (CurrMOps <= DecMOps) ? 0 : CurrMOps - DecMOps;
1828 if ((NextCycle - CurrCycle) > DependentLatency)
1829 DependentLatency = 0;
1831 DependentLatency -= (NextCycle - CurrCycle);
1835 CurrCycle = NextCycle;
1839 for (; CurrCycle != NextCycle; ++CurrCycle) {
1846 CheckPending =
true;
1856 ExecutedResCounts[PIdx] += Count;
1857 if (ExecutedResCounts[PIdx] > MaxExecutedResCount)
1858 MaxExecutedResCount = ExecutedResCounts[PIdx];
1871 unsigned Count = Factor * Cycles;
1873 <<
" +" << Cycles <<
"x" << Factor <<
"u\n");
1883 ZoneCritResIdx = PIdx;
1884 DEBUG(
dbgs() <<
" *** Critical resource "
1890 if (NextAvailable > CurrCycle) {
1893 << NextAvailable <<
"\n");
1895 return NextAvailable;
1915 "Cannot schedule this instruction's MicroOps in the current cycle.");
1918 DEBUG(
dbgs() <<
" Ready @" << ReadyCycle <<
"c\n");
1920 unsigned NextCycle = CurrCycle;
1923 assert(ReadyCycle <= CurrCycle &&
"Broken PendingQueue");
1926 if (ReadyCycle > NextCycle) {
1927 NextCycle = ReadyCycle;
1928 DEBUG(
dbgs() <<
" *** Stall until: " << ReadyCycle <<
"\n");
1937 NextCycle = ReadyCycle;
1940 RetiredMOps += IncMOps;
1947 if (ZoneCritResIdx) {
1949 unsigned ScaledMOps =
1957 DEBUG(
dbgs() <<
" *** Critical resource NumMicroOps: "
1966 if (RCycle > NextCycle)
1977 unsigned PIdx = PI->ProcResourceIdx;
1980 ReservedCycles[PIdx] =
1984 ReservedCycles[PIdx] = NextCycle;
1990 unsigned &TopLatency =
isTop() ? ExpectedLatency : DependentLatency;
1991 unsigned &BotLatency =
isTop() ? DependentLatency : ExpectedLatency;
1995 <<
" TopLatency SU(" << SU->
NodeNum <<
") " << TopLatency <<
"c\n");
2000 <<
" BotLatency SU(" << SU->
NodeNum <<
") " << BotLatency <<
"c\n");
2003 if (NextCycle > CurrCycle) {
2018 CurrMOps += IncMOps;
2020 DEBUG(
dbgs() <<
" *** Max MOps " << CurrMOps
2021 <<
" at cycle " << CurrCycle <<
'\n');
2032 MinReadyCycle = UINT_MAX;
2037 for (
unsigned i = 0, e =
Pending.
size(); i != e; ++i) {
2041 if (ReadyCycle < MinReadyCycle)
2042 MinReadyCycle = ReadyCycle;
2044 if (!IsBuffered && ReadyCycle > CurrCycle)
2055 CheckPending =
false;
2105 if (ZoneCritResIdx) {
2115 <<
" Retired: " << RetiredMOps;
2117 dbgs() <<
"\n Critical: " << ResCount / LFactor <<
"c, "
2118 << ResCount / ResFactor <<
" "
2120 <<
"\n ExpectedLatency: " << ExpectedLatency <<
"c\n"
2121 << (IsResourceLimited ?
" - Resource" :
" - Latency")
2171 RemLatency = std::max(RemLatency,
2173 RemLatency = std::max(RemLatency,
2177 unsigned OtherCritIdx = 0;
2178 unsigned OtherCount =
2181 bool OtherResLimited =
false;
2184 OtherResLimited = (
int)(OtherCount - (RemLatency * LFactor)) > (
int)LFactor;
2189 if (!OtherResLimited) {
2193 <<
" RemainingLatency " << RemLatency <<
" + "
2208 if (OtherResLimited)
2209 dbgs() <<
" RemainingLimit: "
2212 dbgs() <<
" Latency limited both directions.\n");
2217 if (OtherResLimited)
2225 case NoCand:
return "NOCAND ";
2229 case Stall:
return "STALL ";
2230 case Cluster:
return "CLUSTER ";
2231 case Weak:
return "WEAK ";
2232 case RegMax:
return "REG-MAX ";
2247 unsigned ResIdx = 0;
2248 unsigned Latency = 0;
2291 dbgs() <<
" " << Latency <<
" cycles ";
2303 if (TryVal < CandVal) {
2307 if (TryVal > CandVal) {
2308 if (Cand.
Reason > Reason)
2320 if (TryVal > CandVal) {
2324 if (TryVal < CandVal) {
2325 if (Cand.
Reason > Reason)
2361 DEBUG(
dbgs() <<
"Pick " << (IsTop ?
"Top " :
"Bot ")
2367 "(PreRA)GenericScheduler needs vreg liveness");
2381 if (!Top.HazardRec) {
2383 DAG->MF.getSubtarget().getInstrInfo()->CreateTargetMIHazardRecognizer(
2386 if (!Bot.HazardRec) {
2388 DAG->MF.getSubtarget().getInstrInfo()->CreateTargetMIHazardRecognizer(
2396 unsigned NumRegionInstrs) {
2403 RegionPolicy.ShouldTrackPressure =
true;
2409 RegionPolicy.ShouldTrackPressure = NumRegionInstrs > (NIntRegs / 2);
2415 RegionPolicy.OnlyBottomUp =
true;
2423 RegionPolicy.ShouldTrackPressure =
false;
2428 "-misched-topdown incompatible with -misched-bottomup");
2431 if (RegionPolicy.OnlyBottomUp)
2432 RegionPolicy.OnlyTopDown =
false;
2436 if (RegionPolicy.OnlyTopDown)
2437 RegionPolicy.OnlyBottomUp =
false;
2455 unsigned IterCount =
2461 unsigned InFlightCount =
2463 unsigned BufferLimit =
2471 <<
"c NumIters=" << (AcyclicCount + IterCount-1) / IterCount
2475 dbgs() <<
" ACYCLIC LATENCY LIMIT\n");
2482 for (std::vector<SUnit*>::const_iterator
2483 I = Bot.Available.begin(), E = Bot.Available.end(); I != E; ++
I) {
2494 checkAcyclicLatency();
2506 if (TryRank == CandRank) {
2519 return tryGreater(TryRank, CandRank, TryCand, Cand, Reason);
2538 unsigned ScheduledOper = isTop ? 1 : 0;
2539 unsigned UnscheduledOper = isTop ? 0 : 1;
2550 return AtBoundary ? -1 : 1;
2571 if (DAG->isTrackingPressure()) {
2577 DAG->getRegionCriticalPSets(),
2578 DAG->getRegPressure().MaxSetPressure);
2584 &DAG->getPressureDiff(TryCand.
SU),
2586 DAG->getRegionCriticalPSets(),
2587 DAG->getRegPressure().MaxSetPressure);
2592 DAG->getPressureDiff(TryCand.
SU),
2594 DAG->getRegionCriticalPSets(),
2595 DAG->getRegPressure().MaxSetPressure);
2645 const SUnit *NextClusterSU =
2647 if (
tryGreater(TryCand.
SU == NextClusterSU, Cand.SU == NextClusterSU,
2654 TryCand, Cand,
Weak)) {
2659 Cand.RPDelta.CurrentMax,
2669 Cand.ResDelta.DemandedResources,
2713 tryCandidate(Cand, TryCand, Zone, RPTracker, TempTracker);
2728 if (
SUnit *SU = Bot.pickOnlyChoice()) {
2733 if (
SUnit *SU = Top.pickOnlyChoice()) {
2749 pickNodeFromQueue(Bot, DAG->getBotRPTracker(), BotCand);
2750 assert(BotCand.
Reason !=
NoCand &&
"failed to find the first candidate");
2768 pickNodeFromQueue(Top, DAG->getTopRPTracker(), TopCand);
2769 assert(TopCand.
Reason !=
NoCand &&
"failed to find the first candidate");
2786 assert(Top.Available.empty() && Top.Pending.empty() &&
2787 Bot.Available.empty() && Bot.Pending.empty() &&
"ReadyQ garbage");
2792 if (RegionPolicy.OnlyTopDown) {
2793 SU = Top.pickOnlyChoice();
2797 pickNodeFromQueue(Top, DAG->getTopRPTracker(), TopCand);
2798 assert(TopCand.
Reason !=
NoCand &&
"failed to find a candidate");
2804 else if (RegionPolicy.OnlyBottomUp) {
2805 SU = Bot.pickOnlyChoice();
2809 pickNodeFromQueue(Bot, DAG->getBotRPTracker(), BotCand);
2810 assert(BotCand.
Reason !=
NoCand &&
"failed to find a candidate");
2817 SU = pickNodeBidirectional(IsTopNode);
2822 Top.removeReady(SU);
2824 Bot.removeReady(SU);
2843 SUnit *DepSU = I->getSUnit();
2844 if (isTop ? DepSU->
Succs.size() > 1 : DepSU->
Preds.size() > 1)
2849 DEBUG(
dbgs() <<
" Rescheduling physreg copy ";
2850 I->getSUnit()->dump(DAG));
2867 reschedulePhysRegCopies(SU,
true);
2873 reschedulePhysRegCopies(SU,
false);
2914 if (!Top.HazardRec) {
2927 I = BotRoots.begin(), E = BotRoots.end(); I != E; ++
I) {
2951 if (
tryLess(Top.getLatencyStallCycles(TryCand.
SU),
2952 Top.getLatencyStallCycles(Cand.
SU), TryCand, Cand,
Stall))
2960 Cand.ResDelta.DemandedResources,
2965 if (Cand.Policy.ReduceLatency &&
tryLatency(TryCand, Cand, Top)) {
2970 if (TryCand.
SU->
NodeNum < Cand.SU->NodeNum)
2983 tryCandidate(Cand, TryCand);
2994 assert(Top.Available.empty() && Top.Pending.empty() &&
"ReadyQ garbage");
2999 SU = Top.pickOnlyChoice();
3006 pickNodeFromQueue(TopCand);
3007 assert(TopCand.
Reason !=
NoCand &&
"failed to find a candidate");
3014 Top.removeReady(SU);
3029 return new ScheduleDAGMI(C, make_unique<PostGenericScheduler>(C),
true);
3043 ILPOrder(
bool MaxILP)
3044 : DFSResult(nullptr), ScheduledTrees(nullptr), MaximizeILP(MaxILP) {}
3049 bool operator()(
const SUnit *
A,
const SUnit *B)
const {
3050 unsigned SchedTreeA = DFSResult->getSubtreeID(A);
3051 unsigned SchedTreeB = DFSResult->getSubtreeID(B);
3052 if (SchedTreeA != SchedTreeB) {
3054 if (ScheduledTrees->test(SchedTreeA) != ScheduledTrees->test(SchedTreeB))
3055 return ScheduledTrees->test(SchedTreeB);
3058 if (DFSResult->getSubtreeLevel(SchedTreeA)
3059 != DFSResult->getSubtreeLevel(SchedTreeB)) {
3060 return DFSResult->getSubtreeLevel(SchedTreeA)
3061 < DFSResult->getSubtreeLevel(SchedTreeB);
3065 return DFSResult->getILP(A) < DFSResult->getILP(B);
3067 return DFSResult->getILP(A) > DFSResult->getILP(B);
3076 std::vector<SUnit*> ReadyQ;
3078 ILPScheduler(
bool MaximizeILP): DAG(nullptr), Cmp(MaximizeILP) {}
3084 Cmp.DFSResult = DAG->getDFSResult();
3085 Cmp.ScheduledTrees = &DAG->getScheduledTrees();
3091 std::make_heap(ReadyQ.begin(), ReadyQ.end(), Cmp);
3099 if (ReadyQ.empty())
return nullptr;
3100 std::pop_heap(ReadyQ.begin(), ReadyQ.end(), Cmp);
3101 SUnit *SU = ReadyQ.back();
3104 DEBUG(
dbgs() <<
"Pick node " <<
"SU(" << SU->NodeNum <<
") "
3105 <<
" ILP: " << DAG->getDFSResult()->getILP(SU)
3106 <<
" Tree: " << DAG->getDFSResult()->getSubtreeID(SU) <<
" @"
3107 << DAG->getDFSResult()->getSubtreeLevel(
3108 DAG->getDFSResult()->getSubtreeID(SU)) <<
'\n'
3109 <<
"Scheduling " << *SU->getInstr());
3115 std::make_heap(ReadyQ.begin(), ReadyQ.end(), Cmp);
3121 assert(!IsTopNode &&
"SchedDFSResult needs bottom-up");
3127 ReadyQ.push_back(SU);
3128 std::push_heap(ReadyQ.begin(), ReadyQ.end(), Cmp);
3152 template<
bool IsReverse>
3176 InstructionShuffler(
bool alternate,
bool topdown)
3177 : IsAlternating(alternate), IsTopDown(topdown) {}
3191 if (TopQ.empty())
return nullptr;
3199 if (BottomQ.empty())
return nullptr;
3206 IsTopDown = !IsTopDown;
3225 "-misched-topdown incompatible with -misched-bottomup");
3226 return new ScheduleDAGMILive(C, make_unique<InstructionShuffler>(Alternate, TopDown));
3229 "shuffle",
"Shuffle machine instructions alternating directions",
3257 return (Node->
Preds.size() > 10 || Node->
Succs.size() > 10);
3271 return "color=cyan,style=dashed";
3273 return "color=blue,style=dashed";
3293 std::string Str(
"shape=Mrecord");
3298 Str +=
",style=filled,fillcolor=\"#";
3315 errs() <<
"ScheduleDAGMI::viewGraph is only available in debug builds on "
3316 <<
"systems with Graphviz or gv!\n";
3322 viewGraph(getDAGName(),
"Scheduling-Units Graph for " + getDAGName());
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
iterator end()
Returns an iterator past this container.
virtual ScheduleHazardRecognizer * CreateTargetMIHazardRecognizer(const InstrItineraryData *, const ScheduleDAG *DAG) const
Allocate and return a hazard recognizer to use for this target when scheduling the machine instructio...
void computeDFSResult()
Compute a DFSResult after DAG building is complete, and before any queue comparisons.
void releaseSucc(SUnit *SU, SDep *SuccEdge)
ReleaseSucc - Decrement the NumPredsLeft count of a successor.
static int biasPhysRegCopy(const SUnit *SU, bool isTop)
Minimize physical register live ranges.
Weak DAG edge linking a chain of clustered instrs.
void push_back(const T &Elt)
void schedNode(SUnit *SU, bool IsTopNode) override
Called after ScheduleDAGMI has scheduled an instruction and updated scheduled/remaining flags in the ...
const_iterator end(StringRef path)
Get end iterator over path.
virtual void finishBlock()
finishBlock - Clean up after scheduling in the given block.
virtual void initialize(ScheduleDAGMI *DAG)=0
Initialize the strategy after building the DAG for a new region.
bool advance()
Advance across the current instruction.
void pickNodeFromQueue(SchedBoundary &Zone, const RegPressureTracker &RPTracker, SchedCandidate &Candidate)
Pick the best candidate from the queue.
virtual bool enableMachineScheduler() const
True if the subtarget should run MachineScheduler after aggressive coalescing.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static cl::opt< bool > ViewMISchedDAGs("view-misched-dags", cl::Hidden, cl::desc("Pop up a window to show MISched dags after they are processed"))
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static std::string getNodeAttributes(const SUnit *N, const ScheduleDAG *G)
void clear()
Clear the results.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Segments::iterator iterator
bool isArtificialDep() const
virtual ~MachineSchedContext()
Each Scheduling boundary is associated with ready queues.
SlotIndex def
The index of the defining instruction.
void releaseTopNode(SUnit *SU)
void addPressureChange(unsigned RegUnit, bool IsDec, const MachineRegisterInfo *MRI)
Add a change in pressure to the pressure diff of a given instruction.
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
const MachineSchedContext * Context
SUnit * pickNodeBidirectional(bool &IsTopNode)
Pick the best candidate node from either the top or bottom queue.
virtual void releaseTopNode(SUnit *SU)=0
When all predecessor dependencies have been resolved, free this node for top-down scheduling...
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
void init(ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel)
A Module instance is used to store all the information related to an LLVM module. ...
bool isInQueue(SUnit *SU) const
unsigned getCurrCycle() const
Number of cycles to issue the instructions scheduled in this zone.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
static MachineSchedRegistry ILPMaxRegistry("ilpmax","Schedule bottom-up for max ILP", createILPMaxScheduler)
ScheduleDAGTopologicalSort Topo
Topo - A topological ordering for SUnits which permits fast IsReachable and similar queries...
static bool tryGreater(int TryVal, int CandVal, GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, GenericSchedulerBase::CandReason Reason)
MachineBasicBlock::iterator CurrentTop
The top of the unscheduled zone.
void buildSchedGraph(AliasAnalysis *AA, RegPressureTracker *RPTracker=nullptr, PressureDiffs *PDiffs=nullptr)
buildSchedGraph - Build SUnits from the MachineBasicBlock that we are input.
static bool isSchedBoundary(MachineBasicBlock::iterator MI, MachineBasicBlock *MBB, MachineFunction *MF, const TargetInstrInfo *TII, bool IsPostRA)
Return true of the given instruction should not be included in a scheduling region.
char & MachineDominatorsID
MachineDominators - This pass is a machine dominators analysis pass.
LiveInterval - This class represents the liveness of a register, or stack slot.
static cl::opt< std::string > SchedOnlyFunc("misched-only-func", cl::Hidden, cl::desc("Only schedule this function"))
MachineInstr * getInstr() const
getInstr - Return the representative MachineInstr for this SUnit.
SlotIndex getInstructionIndex(const MachineInstr *instr) const
Returns the base index of the given instruction.
unsigned computeCyclicCriticalPath()
Compute the cyclic critical path through the DAG.
static bool isVirtualRegister(unsigned Reg)
isVirtualRegister - Return true if the specified register number is in the virtual register namespace...
void verify(Pass *p=nullptr, const char *Banner=nullptr) const
verify - Run the current MachineFunction through the machine code verifier, useful for debugger use...
void traceCandidate(const SchedCandidate &Cand)
unsigned DemandedResources
static bool renderGraphFromBottomUp()
bool ShouldTrackPressure
Register pressure in this region computed by initRegPressure.
bool isLocal(SlotIndex Start, SlotIndex End) const
True iff this segment is a single segment that lies between the specified boundaries, exclusively.
MachineBasicBlock::iterator begin() const
begin - Return an iterator to the top of the current scheduling region.
unsigned getMicroOpBufferSize() const
Number of micro-ops that may be buffered for OOO execution.
Mutate the DAG as a postpass after normal DAG building.
const_iterator begin(StringRef path)
Get begin iterator over path.
static ScheduleDAGInstrs * createILPMaxScheduler(MachineSchedContext *C)
virtual std::string getGraphNodeLabel(const SUnit *SU) const =0
getGraphNodeLabel - Return a label for an SUnit node in a visualization of the ScheduleDAG.
virtual bool isSchedulingBoundary(const MachineInstr *MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
const MCSchedClassDesc * getSchedClass(SUnit *SU) const
Resolve and cache a resolved scheduling class for an SUnit.
static ScheduleDAGInstrs * createInstructionShuffler(MachineSchedContext *C)
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
char & MachineSchedulerID
MachineScheduler - This pass schedules machine instructions.
void closeRegion()
Finalize the region boundaries and recored live ins and live outs.
Summarize the unscheduled region.
virtual bool enablePostRAScheduler() const
True if the subtarget should run a scheduler after register allocation.
unsigned getResourceFactor(unsigned ResIdx) const
Multiply the number of units consumed for a resource by this factor to normalize it relative to other...
unsigned getNumAllocatableRegs(const TargetRegisterClass *RC) const
getNumAllocatableRegs - Returns the number of actually allocatable registers in RC in the current fun...
RegisterPassParser class - Handle the addition of new machine passes.
MachineSchedRegistry provides a selection of available machine instruction schedulers.
RegisterClassInfo * RegClassInfo
MachineBasicBlock::iterator top() const
VNInfo - Value Number Information.
Instructions::iterator instr_iterator
static std::string getNodeLabel(const SUnit *SU, const ScheduleDAG *G)
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
std::unique_ptr< MachineSchedStrategy > SchedImpl
void fixupKills(MachineBasicBlock *MBB)
Fix register kill flags that scheduling has made invalid.
bool isArtificial() const
isArtificial - Test if this is an Order dependence that is marked as "artificial", meaning it isn't necessary for correctness.
static void tracePick(const GenericSchedulerBase::SchedCandidate &Cand, bool IsTop)
INITIALIZE_PASS_BEGIN(MachineScheduler,"machine-scheduler","Machine Instruction Scheduler", false, false) INITIALIZE_PASS_END(MachineScheduler
virtual void AdvanceCycle()
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
void updateQueues(SUnit *SU, bool IsTopNode)
Update scheduler DAG and queues after scheduling an instruction.
virtual void schedNode(SUnit *SU, bool IsTopNode)=0
Notify MachineSchedStrategy that ScheduleDAGMI has scheduled an instruction and updated scheduled/rem...
SmallVector< SDep, 4 > Preds
virtual void startBlock(MachineBasicBlock *BB)
startBlock - Prepare to perform scheduling in the given block.
MachineBasicBlock::const_iterator getPos() const
Get the MI position corresponding to this register pressure.
A register anti-dependedence (aka WAR).
ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules machine instructions while...
iterator_base< SparseMultiSet * > iterator
unsigned getDependentLatency() const
const TargetSchedModel * getSchedModel() const
Get the machine model for instruction scheduling.
unsigned NumInstrsScheduled
The number of instructions scheduled so far.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
static const SCEV * apply(const SCEV *Scev, LoopToScevMapT &Map, ScalarEvolution &SE)
Applies the Map (Loop -> SCEV) to the given Scev.
unsigned getNumSubtrees() const
The number of subtrees detected in this DAG.
unsigned getHeight() const
getHeight - Return the height of this node, which is the length of the maximum path down to any node ...
ArrayRef< SUnit * > elements()
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
void clear()
clear - Clear all bits.
void releaseBottomNode(SUnit *SU)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineBasicBlock::iterator RegionEnd
The end of the range to be scheduled.
void closeBottom()
Set the boundary for the bottom of the region and summarize live outs.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
bool isWeak() const
isWeak - Test if this a weak dependence.
Provide an instruction scheduling machine model to CodeGen passes.
const HexagonInstrInfo * TII
static std::string getEdgeAttributes(const SUnit *Node, SUnitIterator EI, const ScheduleDAG *Graph)
If you want to override the dot attributes printed for a particular edge, override this method...
static MachineSchedRegistry DefaultSchedRegistry("default","Use the target's default scheduler choice.", useDefaultMachineSched)
static cl::opt< MachineSchedRegistry::ScheduleDAGCtor, false, RegisterPassParser< MachineSchedRegistry > > MachineSchedOpt("misched", cl::init(&useDefaultMachineSched), cl::Hidden, cl::desc("Machine instruction scheduler to use"))
MachineSchedOpt allows command line selection of the scheduler.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
virtual const char * getRegPressureSetName(unsigned Idx) const =0
Get the name of this register unit pressure set.
static unsigned getWeakLeft(const SUnit *SU, bool isTop)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Regular data dependence (aka true-dependence).
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
Result of a LiveRange query.
Reg
All possible values of the reg field in the ModR/M byte.
PressureDiff & getPressureDiff(const SUnit *SU)
std::vector< std::unique_ptr< ScheduleDAGMutation > > Mutations
Ordered list of DAG postprocessing steps.
virtual bool getMemOpBaseRegImmOfs(MachineInstr *MemOp, unsigned &BaseReg, unsigned &Offset, const TargetRegisterInfo *TRI) const
Get the base register and byte offset of an instruction that reads/writes memory. ...
void InitDAGTopologicalSorting()
InitDAGTopologicalSorting - create the initial topological ordering from the DAG to be scheduled...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Target-Independent Code Generator Pass Configuration Options.
bool IsAcyclicLatencyLimited
static ScheduleDAGInstrs * createGenericSchedPostRA(MachineSchedContext *C)
Create a generic scheduler with no vreg liveness or DAG mutation passes.
static MachineSchedRegistry ShufflerRegistry("shuffle","Shuffle machine instructions alternating directions", createInstructionShuffler)
void initPolicy(MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, unsigned NumRegionInstrs) override
Initialize the per-region scheduling policy.
Compute the values of each DAG node for various metrics during DFS.
load Combine Adjacent Loads
void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs) override
Implement the ScheduleDAGInstrs interface for handling the next scheduling region.
static const unsigned InvalidCycle
MachineBasicBlock::iterator LiveRegionEnd
unsigned getCriticalCount() const
Get the scaled count of scheduled micro-ops and resources, including executed resources.
static cl::opt< bool > VerifyScheduling("verify-misched", cl::Hidden, cl::desc("Verify machine instrs before and after machine scheduling"))
void bumpCycle(unsigned NextCycle)
Move the boundary of scheduled code by one cycle.
unsigned countResource(unsigned PIdx, unsigned Cycles, unsigned ReadyCycle)
Add the given processor resource to this scheduled zone.
static MachineBasicBlock::const_iterator priorNonDebug(MachineBasicBlock::const_iterator I, MachineBasicBlock::const_iterator Beg)
Decrement this iterator until reaching the top or a non-debug instr.
void scheduleMI(SUnit *SU, bool IsTopNode)
Move an instruction and update register pressure.
void tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand, SchedBoundary &Zone, const RegPressureTracker &RPTracker, RegPressureTracker &TempTracker)
Apply a set of heursitics to a new candidate.
MachineBasicBlock::iterator RegionBegin
The beginning of the range to be scheduled.
reverse_iterator rbegin() const
RegPressureTracker BotRPTracker
void buildDAGWithRegPressure()
Call ScheduleDAGInstrs::buildSchedGraph with register pressure tracking enabled.
bool recede(SmallVectorImpl< unsigned > *LiveUses=nullptr, PressureDiff *PDiff=nullptr)
Recede across the previous instruction.
virtual void RecedeCycle()
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
VReg2UseMap VRegUses
After calling BuildSchedGraph, each vreg used in the scheduling region is mapped to a set of SUnits...
std::vector< PressureChange > RegionCriticalPSets
List of pressure sets that exceed the target's pressure limit before scheduling, listed in increasing...
static const char * getReasonStr(GenericSchedulerBase::CandReason Reason)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
static std::string getGraphName(const ScheduleDAG *G)
Itinerary data supplied by a subtarget to be used by a target.
void print(raw_ostream &OS, SlotIndexes *=nullptr) const
print - Print out the MachineFunction in a format suitable for debugging to the specified stream...
void releasePending()
Release pending ready nodes in to the available queue.
COFF::MachineTypes Machine
MachinePassRegistry - Track the registration of machine passes.
virtual void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs)
Initialize the scheduler state for the next scheduling region.
void bumpNode(SUnit *SU)
Move the boundary of scheduled code by one SUnit.
SUnit * pickNode(bool &IsTopNode) override
Pick the next node to schedule.
CandReason
Represent the type of SchedCandidate found within a single queue.
size_t size() const
size - Get the array size.
virtual bool enableClusterLoads() const
void checkAcyclicLatency()
Set IsAcyclicLatencyLimited if the acyclic path is longer than the cyclic critical path by more cycle...
static std::string getNodeDescription(const SUnit *SU, const ScheduleDAG *G)
void incExecutedResources(unsigned PIdx, unsigned Count)
TargetInstrInfo - Interface to description of machine instruction set.
virtual void registerRoots()
Notify this strategy that all roots have been released (including those that depend on EntrySU or Exi...
static cl::opt< bool > EnableLoadCluster("misched-cluster", cl::Hidden, cl::desc("Enable load clustering."), cl::init(true))
void resize(unsigned NumSUnits)
Initialize the result data with the size of the DAG.
std::vector< SUnit * >::iterator iterator
SDep - Scheduling dependency.
reverse_iterator rend() const
RegisterClassInfo * RegClassInfo
bundle_iterator< MachineInstr, instr_iterator > iterator
void getMaxDownwardPressureDelta(const MachineInstr *MI, RegPressureDelta &Delta, ArrayRef< PressureChange > CriticalPSets, ArrayRef< unsigned > MaxPressureLimit)
Consider the pressure increase caused by traversing this instruction top-down.
iterator remove(iterator I)
virtual void overrideSchedPolicy(MachineSchedPolicy &Policy, MachineInstr *begin, MachineInstr *end, unsigned NumRegionInstrs) const
Override generic scheduling policy within a region.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
initializer< Ty > init(const Ty &Val)
ScheduleHazardRecognizer * HazardRec
unsigned getNextResourceCycle(unsigned PIdx, unsigned Cycles)
Compute the next cycle at which the given processor resource can be scheduled.
LiveQueryResult Query(SlotIndex Idx) const
Query Liveness at Idx.
ArrayRef< unsigned > getLiveThru() const
Helpers for implementing custom MachineSchedStrategy classes.
PressureChange CurrentMax
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
unsigned getScheduledLatency() const
Get the number of latency cycles "covered" by the scheduled instructions.
const InstrItineraryData * getInstrItineraries() const
static const unsigned MinSubtreeSize
Identify one of the processor resource kinds consumed by a particular scheduling class for the specif...
void dumpSchedule() const
dump the scheduled Sequence.
unsigned getZoneCritResIdx() const
void removeReady(SUnit *SU)
Remove SU from the ready set for this boundary.
bundle_iterator - MachineBasicBlock iterator that automatically skips over MIs that are inside bundle...
const MachineOperand & getOperand(unsigned i) const
Summarize the scheduling resources required for an instruction of a particular scheduling class...
unsigned getLatencyFactor() const
Multiply cycle count by this factor to normalize it relative to other resources.
static MachineSchedRegistry GenericSchedRegistry("converge","Standard converging scheduler.", createGenericSchedLive)
RegisterPressure computed within a region of instructions delimited by TopPos and BottomPos...
bool addEdge(SUnit *SuccSU, const SDep &PredDep)
Add a DAG edge to the given SU with the given predecessor dependence data.
Represent the analysis usage information of a pass.
static MachinePassRegistry Registry
unsigned getLatency() const
getLatency - Return the latency value for this edge, which roughly means the minimum number of cycles...
Store the state used by GenericScheduler heuristics, required for the lifetime of one invocation of p...
virtual bool shouldClusterLoads(MachineInstr *FirstLdSt, MachineInstr *SecondLdSt, unsigned NumLoads) const
bool checkHazard(SUnit *SU)
Does this SU have a hazard within the current instruction group.
void releasePred(SUnit *SU, SDep *PredEdge)
ReleasePred - Decrement the NumSuccsLeft count of a predecessor.
Track the current register pressure at some position in the instruction stream, and remember the high...
virtual void releaseBottomNode(SUnit *SU)=0
When all successor dependencies have been resolved, free this node for bottom-up scheduling.
void findRootsAndBiasEdges(SmallVectorImpl< SUnit * > &TopRoots, SmallVectorImpl< SUnit * > &BotRoots)
SmallVector< unsigned, 8 > LiveOutRegs
Policy for scheduling the next instruction in the candidate's zone.
const TargetSchedModel * SchedModel
static ScheduleDAGInstrs * useDefaultMachineSched(MachineSchedContext *C)
A dummy default scheduler factory indicates whether the scheduler is overridden on the command line...
List of PressureChanges in order of increasing, unique PSetID.
virtual void exitRegion()
Notify that the scheduler has finished scheduling the current region.
unsigned getRegPressureSetLimit(unsigned Idx) const
Get the register unit limit for the given pressure set index.
void ViewGraph(const GraphType &G, const Twine &Name, bool ShortNames=false, const Twine &Title="", GraphProgram::Name Program=GraphProgram::DOT)
ViewGraph - Emit a dot graph, run 'dot', run gv on the postscript file, then cleanup.
DOTGraphTraits(bool isSimple=false)
bool empty() const
empty - Check if the array is empty.
SchedResourceDelta ResDelta
const TargetRegisterInfo * TRI
SchedDFSResult * DFSResult
Information about DAG subtrees.
PressureDiffs SUPressureDiffs
void dumpScheduledState()
unsigned getNumProcResourceKinds() const
Get the number of kinds of resources for this target.
static MachineBasicBlock::const_iterator nextIfDebug(MachineBasicBlock::const_iterator I, MachineBasicBlock::const_iterator End)
If this iterator is a debug value, increment until reaching the End or a non-debug instruction...
#define INITIALIZE_AG_DEPENDENCY(depName)
LiveIntervals * getLIS() const
Expose LiveIntervals for use in DAG mutators and such.
bool isNextSU(const SUnit *SU) const
Return true if the given SU is used by the most recently scheduled instruction.
static cl::opt< bool > EnableMacroFusion("misched-fusion", cl::Hidden, cl::desc("Enable scheduling for macro fusion."), cl::init(true))
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool isPHIDef() const
Returns true if this value is defined by a PHI instruction (or was, PHI instructions may have been el...
void releaseSuccessors(SUnit *SU)
releaseSuccessors - Call releaseSucc on each of SU's successors.
const SUnit * NextClusterSucc
std::string & str()
Flushes the stream contents to the target string and returns the string's reference.
void viewGraph() override
Out-of-line implementation with no arguments is handy for gdb.
void setPolicy(CandPolicy &Policy, bool IsPostRA, SchedBoundary &CurrZone, SchedBoundary *OtherZone)
Set the CandPolicy given a scheduling zone given the current resources and latencies inside and outsi...
void initLiveThru(const RegPressureTracker &RPTracker)
Initialize the LiveThru pressure set based on the untied defs found in RPTracker. ...
static cl::opt< bool > EnableRegPressure("misched-regpressure", cl::Hidden, cl::desc("Enable register pressure scheduling."), cl::init(true))
ProcResIter getWriteProcResEnd(const MCSchedClassDesc *SC) const
iterator find(const KeyT &Key)
Find an element by its key.
iterator find(SlotIndex Pos)
find - Return an iterator pointing to the first segment that ends after Pos, or end().
DOTGraphTraits - Template class that can be specialized to customize how graphs are converted to 'dot...
void setRepeat(CandReason R)
void init(ScheduleDAGMI *dag, const TargetSchedModel *smodel, SchedRemainder *rem)
bool hasInstrSchedModel() const
Return true if this machine model includes an instruction-level scheduling model. ...
static cl::opt< unsigned > SchedOnlyBlock("misched-only-block", cl::Hidden, cl::desc("Only schedule this MBB#"))
std::vector< unsigned > MaxSetPressure
Map of max reg pressure indexed by pressure set ID, not class ID.
virtual void EmitInstruction(SUnit *)
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
const SUnit * NextClusterPred
Record the next node in a scheduled cluster.
const MCProcResourceDesc * getProcResource(unsigned PIdx) const
Get a processor resource by ID for convenience.
static bool isSameInstr(SlotIndex A, SlotIndex B)
isSameInstr - Return true if A and B refer to the same instruction.
void compute(ArrayRef< SUnit > SUnits)
Compute various metrics for the DAG with given roots.
void registerRoots() override
Notify this strategy that all roots have been released (including those that depend on EntrySU or Exi...
void reschedulePhysRegCopies(SUnit *SU, bool isTop)
unsigned getExecutedCount() const
Get a scaled count for the minimum execution time of the scheduled micro-ops that are ready to execut...
AnalysisUsage & addRequiredID(const void *ID)
ScheduleDAGInstrs *(* ScheduleDAGCtor)(MachineSchedContext *)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
static ScheduleDAGInstrs * createGenericSchedLive(MachineSchedContext *C)
Forward declare the standard machine scheduler.
static cl::opt< unsigned > MISchedCutoff("misched-cutoff", cl::Hidden, cl::desc("Stop scheduling after N instructions"), cl::init(~0U))
static ScheduleDAGInstrs * createILPMinScheduler(MachineSchedContext *C)
bool test(unsigned Idx) const
bool isCtrlDep() const
isCtrlDep - Test if this is not an SDep::Data dependence.
virtual const TargetLowering * getTargetLowering() const
static bool tryPressure(const PressureChange &TryP, const PressureChange &CandP, GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, GenericSchedulerBase::CandReason Reason)
static cl::opt< bool > EnableMachineSched("enable-misched", cl::desc("Enable the machine instruction scheduling pass."), cl::init(true), cl::Hidden)
std::reverse_iterator< const_iterator > const_reverse_iterator
void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs) override
Implement the ScheduleDAGInstrs interface for handling the next scheduling region.
bool isSuccessor(const MachineBasicBlock *MBB) const
isSuccessor - Return true if the specified MBB is a successor of this block.
cl::opt< bool > DumpCriticalPathLength("misched-dcpl", cl::Hidden, cl::desc("Print critical path length to stdout"))
void releaseNode(SUnit *SU, unsigned ReadyCycle)
CHAIN = SC CHAIN, Imm128 - System call.
StringRef getColorString(unsigned NodeNumber)
Get a color string for this node number.
void updateScheduledPressure(const SUnit *SU, const std::vector< unsigned > &NewMaxPressure)
void setPreservesCFG()
This function should be called by the pass, iff they do not:
LiveInterval & getInterval(unsigned Reg)
static cl::opt< bool > EnableCyclicPath("misched-cyclicpath", cl::Hidden, cl::desc("Enable cyclic critical path analysis."), cl::init(true))
void initResourceDelta(const ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel)
bool isRepeat(CandReason R)
~ScheduleDAGMILive() override
void initializeMachineSchedulerPass(PassRegistry &)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
unsigned getOtherResourceCount(unsigned &OtherCritIdx)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static bool tryLess(int TryVal, int CandVal, GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, GenericSchedulerBase::CandReason Reason)
Return true if this heuristic determines order.
virtual void finalizeSchedule()
finalizeSchedule - Allow targets to perform final scheduling actions at the level of the whole Machin...
void tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand)
Apply a set of heursitics to a new candidate for PostRA scheduling.
bool IsReachable(const SUnit *SU, const SUnit *TargetSU)
IsReachable - Checks if SU is reachable from TargetSU.
void biasCriticalPath()
Order this node's predecessor edges such that the critical path edge occurs first.
PressureChange CriticalMax
virtual void Reset()
Reset - This callback is invoked when a new block of instructions is about to be schedule.
virtual void schedule()=0
schedule - Order nodes according to selected style, filling in the Sequence member.
machine Machine Instruction Scheduler
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
MachineBasicBlock::iterator bottom() const
void moveInstruction(MachineInstr *MI, MachineBasicBlock::iterator InsertPos)
Change the position of an instruction within the basic block and update live ranges and region bounda...
MachineBasicBlock::iterator end() const
end - Return an iterator to the bottom of the current scheduling region.
virtual bool hasVRegLiveness() const
Return true if this DAG supports VReg liveness and RegPressure.
PrintVRegOrUnit - It is often convenient to track virtual registers and physical register units in th...
bool isBoundaryNode() const
Boundary nodes are placeholders for the boundary of the scheduling region.
unsigned getDepth() const
getDepth - Return the depth of this node, which is the length of the maximum path up to any node whic...
void updatePressureDiffs(ArrayRef< unsigned > LiveUses)
Update the PressureDiff array for liveness after scheduling this instruction.
void schedNode(SUnit *SU, bool IsTopNode) override
Update the scheduler's state after scheduling a node.
const SUnit * getNextClusterSucc() const
virtual void scheduleTree(unsigned SubtreeID)
Scheduler callback to notify that a new subtree is scheduled.
void init(const MachineFunction *mf, const RegisterClassInfo *rci, const LiveIntervals *lis, const MachineBasicBlock *mbb, MachineBasicBlock::const_iterator pos, bool ShouldTrackUntiedDefs=false)
Setup the RegPressureTracker.
void closeTop()
Set the boundary for the top of the region and summarize live ins.
void releasePredecessors(SUnit *SU)
releasePredecessors - Call releasePred on each of SU's predecessors.
unsigned getSubtreeID(const SUnit *SU) const
Get the ID of the subtree the given DAG node belongs to.
unsigned getUnscheduledLatency(SUnit *SU) const
void initialize(ScheduleDAGMI *dag) override
Initialize the strategy after building the DAG for a new region.
SlotIndex beginIndex() const
beginIndex - Return the lowest numbered slot covered.
bool isCluster() const
isCluster - Test if this is an Order dependence that is marked as "cluster", meaning it is artificial...
SUnit * getSUnit(MachineInstr *MI) const
getSUnit - Return an existing SUnit for this MI, or NULL.
virtual SUnit * pickNode(bool &IsTopNode)=0
Pick the next node to schedule, or return NULL.
unsigned getNumMicroOps(const MachineInstr *MI, const MCSchedClassDesc *SC=nullptr) const
Return the number of issue slots required for this MI.
ScheduleDAGInstrs - A ScheduleDAG subclass for scheduling lists of MachineInstrs. ...
Representation of each machine instruction.
void initializePostMachineSchedulerPass(PassRegistry &)
bundle_iterator< const MachineInstr, const_instr_iterator > const_iterator
static bool isPhysicalRegister(unsigned Reg)
isPhysicalRegister - Return true if the specified register number is in the physical register namespa...
machine Machine Instruction false
void initialize(ScheduleDAGMI *Dag) override
Initialize the strategy after building the DAG for a new region.
const TargetRegisterInfo * TRI
SUnit * pickOnlyChoice()
Call this before applying any other heuristics to the Available queue.
void dumpRegSetPressure(ArrayRef< unsigned > SetPressure, const TargetRegisterInfo *TRI)
Status of an instruction's critical resource consumption.
~ScheduleDAGMI() override
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
char & PostMachineSchedulerID
PostMachineScheduler - This pass schedules machine instructions postRA.
cl::opt< bool > ForceBottomUp
unsigned findMaxLatency(ArrayRef< SUnit * > ReadySUs)
SlotIndex endIndex() const
endNumber - return the maximum point of the range of the whole, exclusive.
unsigned getNumInstrs(const SUnit *SU) const
Get the number of instructions in the given subtree and its children.
MachineSchedContext provides enough context from the MachineScheduler pass for the target to instanti...
MachineSchedStrategy - Interface to the scheduling algorithm used by ScheduleDAGMI.
Capture a change in pressure for a single pressure set.
void getMaxUpwardPressureDelta(const MachineInstr *MI, PressureDiff *PDiff, RegPressureDelta &Delta, ArrayRef< PressureChange > CriticalPSets, ArrayRef< unsigned > MaxPressureLimit)
Consider the pressure increase caused by traversing this instruction bottom-up.
SmallVector< unsigned, 16 > RemainingCounts
static bool hasNodeAddressLabel(const SUnit *Node, const ScheduleDAG *Graph)
void placeDebugValues()
Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues.
unsigned getLatencyStallCycles(SUnit *SU)
Get the difference between the given SUnit's ready time and the current cycle.
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
SUnit * pickNode(bool &IsTopNode) override
Pick the best node to balance the schedule. Implements MachineSchedStrategy.
const TargetInstrInfo * TII
static bool isNodeHidden(const SUnit *Node)
unsigned getPSetOrMax() const
void registerRoots() override
Notify this strategy that all roots have been released (including those that depend on EntrySU or Exi...
unsigned getReg() const
getReg - Returns the register number.
MachineBasicBlock::iterator CurrentBottom
The bottom of the unscheduled zone.
void addLiveRegs(ArrayRef< unsigned > Regs)
Force liveness of virtual registers or physical register units.
bool addPred(const SDep &D, bool Required=true)
addPred - This adds the specified edge as a pred of the current node if not already.
bool operator<(int64_t V1, const APSInt &V2)
A raw_ostream that writes to an std::string.
void initQueues(ArrayRef< SUnit * > TopRoots, ArrayRef< SUnit * > BotRoots)
Release ExitSU predecessors and setup scheduler queues.
virtual const TargetInstrInfo * getInstrInfo() const
unsigned getMicroOpFactor() const
Multiply number of micro-ops by this factor to normalize it relative to other resources.
void postprocessDAG()
Apply each ScheduleDAGMutation step in order.
SmallVector< SDep, 4 > Succs
unsigned getIssueWidth() const
Maximum number of micro-ops that may be scheduled per cycle.
MachineInstr * getInstructionFromIndex(SlotIndex index) const
Returns the instruction associated with the given index.
bool isBottomReady() const
SmallVector< unsigned, 8 > LiveInRegs
List of live in virtual registers or physical register units.
static MachineSchedRegistry ILPMinRegistry("ilpmin","Schedule bottom-up for min ILP", createILPMinScheduler)
Arbitrary strong DAG edge (no real dependence).
void scheduleTree(unsigned SubtreeID)
Scheduler callback to update SubtreeConnectLevels when a tree is initially scheduled.
BasicBlockListType::iterator iterator
const SUnit * getNextClusterPred() const
This class implements an extremely fast bulk output stream that can only output to a stream...
StringRef getName() const
DefaultDOTGraphTraits - This class provides the default implementations of all of the DOTGraphTraits ...
void AddPred(SUnit *Y, SUnit *X)
AddPred - Updates the topological ordering to accommodate an edge to be added from SUnit X to SUnit Y...
MachineInstr * FirstDbgValue
MachineBasicBlock * BB
State specific to the current scheduling region.
unsigned getCurrMOps() const
Micro-ops issued in the current cycle.
bool canAddEdge(SUnit *SuccSU, SUnit *PredSU)
True if an edge can be added from PredSU to SuccSU without creating a cycle.
const TargetSchedModel * SchedModel
INITIALIZE_PASS(PostMachineScheduler,"postmisched","PostRA Machine Instruction Scheduler", false, false) PostMachineScheduler
void setPos(MachineBasicBlock::const_iterator Pos)
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
VNInfo * valueIn() const
Return the value that is live-in to the instruction.
RegisterPressure & getPressure()
Get the resulting register pressure over the traversed region.
virtual bool shouldScheduleAdjacent(MachineInstr *First, MachineInstr *Second) const
Can this target fuse the given instructions if they are scheduled adjacent.
MachineRegisterInfo & MRI
std::vector< SUnit > SUnits
PriorityQueue - This class behaves like std::priority_queue and provides a few additional convenience...
ProcResIter getWriteProcResBegin(const MCSchedClassDesc *SC) const
SlotIndex - An opaque wrapper around machine indexes.
const char * getResourceName(unsigned PIdx) const
static bool tryLatency(GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary &Zone)
RegPressureTracker TopRPTracker
RegPressureTracker RPTracker
void setBest(SchedCandidate &Best)
virtual HazardType getHazardType(SUnit *m, int Stalls=0)
getHazardType - Return the hazard type of emitting this node.
void pickNodeFromQueue(SchedCandidate &Cand)
unsigned getResourceCount(unsigned ResIdx) const
void dump(const ScheduleDAG *G) const
SUnit - Scheduling unit.
void getUpwardPressureDelta(const MachineInstr *MI, PressureDiff &PDiff, RegPressureDelta &Delta, ArrayRef< PressureChange > CriticalPSets, ArrayRef< unsigned > MaxPressureLimit) const
This is the fast version of querying register pressure that does not directly depend on current liven...
VNInfo * getVNInfoBefore(SlotIndex Idx) const
getVNInfoBefore - Return the VNInfo that is live up to but not necessarilly including Idx...
SUnit - Scheduling unit. This is a node in the scheduling DAG.
void addMutation(std::unique_ptr< ScheduleDAGMutation > Mutation)
Add a postprocessing step to the DAG builder.
cl::opt< bool > ForceTopDown
bool isResourceLimited() const