36 #define DEBUG_TYPE "misched"
40 cl::desc(
"Force top-down list scheduling"));
42 cl::desc(
"Force bottom-up list scheduling"));
45 cl::desc(
"Print critical path length to stdout"));
50 cl::desc(
"Pop up a window to show MISched dags after they are processed"));
55 cl::desc(
"Hide nodes with more predecessor/successor than cutoff"));
61 cl::desc(
"Only schedule this function"));
63 cl::desc(
"Only schedule this MBB#"));
80 cl::desc(
"Enable memop clustering."),
88 cl::desc(
"Verify machine instrs before and after machine scheduling"));
94 void MachineSchedStrategy::anchor() {}
95 void ScheduleDAGMutation::anchor() {}
102 MF(nullptr), MLI(nullptr), MDT(nullptr), PassConfig(nullptr), AA(nullptr), LIS(nullptr) {
124 class MachineScheduler :
public MachineSchedulerBase {
139 class PostMachineScheduler :
public MachineSchedulerBase {
141 PostMachineScheduler();
159 "Machine Instruction Scheduler",
false,
false)
166 MachineScheduler::MachineScheduler()
167 : MachineSchedulerBase(
ID) {
171 void MachineScheduler::getAnalysisUsage(
AnalysisUsage &AU)
const {
189 "PostRA Machine Instruction Scheduler",
false,
false)
191 PostMachineScheduler::PostMachineScheduler()
192 : MachineSchedulerBase(
ID) {
196 void PostMachineScheduler::getAnalysisUsage(
AnalysisUsage &AU)
const {
217 cl::desc(
"Machine instruction scheduler to use"));
225 cl::desc(
"Enable the machine instruction scheduling pass."),
cl::init(
true),
229 "enable-post-misched",
230 cl::desc(
"Enable the post-ra machine instruction scheduling pass."),
237 assert(I != Beg &&
"reached the top of the region, cannot decrement");
239 if (!I->isDebugValue())
258 for(; I !=
End; ++
I) {
259 if (!I->isDebugValue())
332 MLI = &getAnalysis<MachineLoopInfo>();
333 MDT = &getAnalysis<MachineDominatorTree>();
334 PassConfig = &getAnalysis<TargetPassConfig>();
335 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
337 LIS = &getAnalysis<LiveIntervals>();
341 MF->verify(
this,
"Before machine scheduling.");
343 RegClassInfo->runOnMachineFunction(*MF);
347 std::unique_ptr<ScheduleDAGInstrs>
Scheduler(createMachineScheduler());
348 scheduleRegions(*Scheduler,
false);
352 MF->verify(
this,
"After machine scheduling.");
364 DEBUG(
dbgs() <<
"Subtarget disables post-MI-sched.\n");
371 PassConfig = &getAnalysis<TargetPassConfig>();
374 MF->
verify(
this,
"Before post machine scheduling.");
378 std::unique_ptr<ScheduleDAGInstrs>
Scheduler(createPostMachineScheduler());
379 scheduleRegions(*Scheduler,
true);
382 MF->verify(
this,
"After post machine scheduling.");
442 if (RegionEnd !=
MBB->
end() ||
449 unsigned NumRegionInstrs = 0;
463 if (I == RegionEnd || I == std::prev(RegionEnd)) {
469 DEBUG(
dbgs() <<
"********** MI Scheduling **********\n");
472 <<
"\n From: " << *I <<
" To: ";
473 if (RegionEnd !=
MBB->
end())
dbgs() << *RegionEnd;
474 else dbgs() <<
"End";
475 dbgs() <<
" RegionInstrs: " << NumRegionInstrs <<
'\n');
477 errs() << MF->getName();
491 RegionEnd = Scheduler.
begin();
509 dbgs() <<
"Queue " << Name <<
": ";
510 for (
unsigned i = 0, e = Queue.size();
i < e; ++
i)
511 dbgs() << Queue[
i]->NodeNum <<
" ";
557 dbgs() <<
"*** Scheduling failed! ***\n";
559 dbgs() <<
" has been released too many times!\n";
596 dbgs() <<
"*** Scheduling failed! ***\n";
598 dbgs() <<
" has been released too many times!\n";
627 unsigned regioninstrs)
631 SchedImpl->initPolicy(begin, end, regioninstrs);
670 DEBUG(
dbgs() <<
"ScheduleDAGMI::schedule starting\n");
690 for (
unsigned su = 0, e =
SUnits.size(); su != e; ++su)
700 bool IsTopNode =
false;
702 DEBUG(
dbgs() <<
"** ScheduleDAGMI::schedule picking next node\n");
743 unsigned BBNum =
begin()->getParent()->getNumber();
744 dbgs() <<
"*** Final schedule for BB#" << BBNum <<
" ***\n";
752 for (
unsigned i = 0, e =
Mutations.size();
i < e; ++
i) {
760 for (std::vector<SUnit>::iterator
769 if (!I->NumPredsLeft)
772 if (!I->NumSuccsLeft)
789 I = TopRoots.
begin(),
E = TopRoots.
end(); I !=
E; ++
I) {
828 for (std::vector<std::pair<MachineInstr *, MachineInstr *> >::iterator
830 std::pair<MachineInstr *, MachineInstr *>
P = *std::prev(DI);
843 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
849 dbgs() <<
"Missing SUnit\n";
873 unsigned Reg = MO.getReg();
879 bool FoundDef =
false;
881 if (MO2.isReg() && MO2.isDef() && MO2.getReg() == Reg && !MO2.isDead()) {
908 unsigned regioninstrs)
922 "ShouldTrackLaneMasks requires ShouldTrackPressure");
972 dbgs() <<
"Top Pressure:\n";
974 dbgs() <<
"Bottom Pressure:\n";
985 for (
unsigned i = 0, e = RegionPressure.size();
i < e; ++
i) {
987 if (RegionPressure[
i] > Limit) {
989 <<
" Limit " << Limit
990 <<
" Actual " << RegionPressure[
i] <<
"\n");
1003 const std::vector<unsigned> &NewMaxPressure) {
1010 unsigned ID = I->getPSet();
1015 && NewMaxPressure[ID] <= INT16_MAX)
1019 if (NewMaxPressure[ID] >= Limit - 2) {
1021 << NewMaxPressure[
ID]
1022 << ((NewMaxPressure[
ID] > Limit) ?
" > " :
" <= ") << Limit
1033 unsigned Reg =
P.RegUnit;
1043 bool Decrement =
P.LaneMask.any();
1047 SUnit &SU = *V2SU.SU;
1054 dbgs() <<
" UpdateRegP: SU(" << SU.
NodeNum <<
") "
1079 assert(VNI &&
"No live value at use.");
1082 SUnit *SU = V2SU.SU;
1092 dbgs() <<
" UpdateRegP: SU(" << SU->
NodeNum <<
") "
1115 DEBUG(
dbgs() <<
"ScheduleDAGMILive::schedule starting\n");
1136 dbgs() <<
" Pressure Diff : ";
1149 bool IsTopNode =
false;
1151 DEBUG(
dbgs() <<
"** ScheduleDAGMILive::schedule picking next node\n");
1180 unsigned BBNum =
begin()->getParent()->getNumber();
1181 dbgs() <<
"*** Final schedule for BB#" << BBNum <<
" ***\n";
1252 unsigned MaxCyclicLatency = 0;
1255 unsigned Reg =
P.RegUnit;
1268 unsigned LiveOutHeight = DefSU->
getHeight();
1273 SUnit *SU = V2SU.SU;
1285 unsigned CyclicLatency = 0;
1287 CyclicLatency = LiveOutDepth - SU->
getDepth();
1290 if (LiveInHeight > LiveOutHeight) {
1291 if (LiveInHeight - LiveOutHeight < CyclicLatency)
1292 CyclicLatency = LiveInHeight - LiveOutHeight;
1297 << SU->
NodeNum <<
") = " << CyclicLatency <<
"c\n");
1298 if (CyclicLatency > MaxCyclicLatency)
1299 MaxCyclicLatency = CyclicLatency;
1302 DEBUG(
dbgs() <<
"Cyclic Critical Path: " << MaxCyclicLatency <<
"c\n");
1303 return MaxCyclicLatency;
1347 dbgs() <<
"Top Pressure:\n";
1357 if (&*priorII == MI)
1384 dbgs() <<
"Bottom Pressure:\n";
1406 MemOpInfo(
SUnit *su,
unsigned reg, int64_t ofs)
1407 : SU(su), BaseReg(reg),
Offset(ofs) {}
1409 bool operator<(
const MemOpInfo&RHS)
const {
1410 return std::tie(BaseReg,
Offset, SU->NodeNum) <
1411 std::tie(RHS.BaseReg, RHS.Offset, RHS.SU->NodeNum);
1422 : TII(tii), TRI(tri), IsLoad(IsLoad) {}
1430 class StoreClusterMutation :
public BaseMemOpClusterMutation {
1434 : BaseMemOpClusterMutation(tii, tri,
false) {}
1437 class LoadClusterMutation :
public BaseMemOpClusterMutation {
1440 : BaseMemOpClusterMutation(tii, tri,
true) {}
1446 std::unique_ptr<ScheduleDAGMutation>
1453 std::unique_ptr<ScheduleDAGMutation>
1462 void BaseMemOpClusterMutation::clusterNeighboringMemOps(
1465 for (
unsigned Idx = 0,
End = MemOps.
size(); Idx !=
End; ++Idx) {
1466 SUnit *SU = MemOps[Idx];
1470 MemOpRecords.
push_back(MemOpInfo(SU, BaseReg, Offset));
1472 if (MemOpRecords.
size() < 2)
1475 std::sort(MemOpRecords.
begin(), MemOpRecords.
end());
1476 unsigned ClusterLength = 1;
1477 for (
unsigned Idx = 0,
End = MemOpRecords.
size(); Idx < (
End - 1); ++Idx) {
1478 if (MemOpRecords[Idx].BaseReg != MemOpRecords[Idx+1].BaseReg) {
1483 SUnit *SUa = MemOpRecords[Idx].SU;
1484 SUnit *SUb = MemOpRecords[Idx+1].SU;
1495 SI = SUa->
Succs.begin(), SE = SUa->
Succs.end();
SI != SE; ++
SI) {
1496 if (
SI->getSUnit() == SUb)
1498 DEBUG(
dbgs() <<
" Copy Succ SU(" <<
SI->getSUnit()->NodeNum <<
")\n");
1516 for (
unsigned Idx = 0,
End = DAG->
SUnits.size(); Idx !=
End; ++Idx) {
1522 unsigned ChainPredID = DAG->
SUnits.size();
1524 PI = SU->
Preds.begin(), PE = SU->
Preds.end(); PI != PE; ++PI) {
1526 ChainPredID = PI->getSUnit()->NodeNum;
1532 unsigned NumChains = StoreChainDependents.
size();
1533 std::pair<DenseMap<unsigned, unsigned>::iterator,
bool> Result =
1534 StoreChainIDs.
insert(std::make_pair(ChainPredID, NumChains));
1536 StoreChainDependents.
resize(NumChains + 1);
1537 StoreChainDependents[Result.first->second].
push_back(SU);
1541 for (
unsigned Idx = 0,
End = StoreChainDependents.
size(); Idx !=
End; ++Idx)
1542 clusterNeighboringMemOps(StoreChainDependents[Idx], DAG);
1564 std::unique_ptr<ScheduleDAGMutation>
1598 assert(Success &&
"No DAG nodes should be reachable from ExitSU");
1641 std::unique_ptr<ScheduleDAGMutation>
1644 return make_unique<CopyConstrain>(
TII, TRI);
1674 unsigned SrcReg = SrcOp.
getReg();
1679 unsigned DstReg = DstOp.
getReg();
1689 unsigned LocalReg = SrcReg;
1690 unsigned GlobalReg = DstReg;
1692 if (!LocalLI->
isLocal(RegionBeginIdx, RegionEndIdx)) {
1696 if (!LocalLI->
isLocal(RegionBeginIdx, RegionEndIdx))
1707 if (GlobalSegment == GlobalLI->
end())
1714 if (GlobalSegment->contains(LocalLI->
beginIndex()))
1717 if (GlobalSegment == GlobalLI->
end())
1721 if (GlobalSegment != GlobalLI->
begin()) {
1724 GlobalSegment->start)) {
1735 assert(std::prev(GlobalSegment)->start < LocalLI->beginIndex() &&
1736 "Disconnected LRG within the scheduling region.");
1753 I = LastLocalSU->
Succs.begin(),
E = LastLocalSU->
Succs.end();
1755 if (I->getKind() !=
SDep::Data || I->getReg() != LocalReg)
1757 if (I->getSUnit() == GlobalSU)
1759 if (!DAG->
canAddEdge(GlobalSU, I->getSUnit()))
1770 I = GlobalSU->
Preds.begin(),
E = GlobalSU->
Preds.end(); I !=
E; ++
I) {
1771 if (I->getKind() !=
SDep::Anti || I->getReg() != GlobalReg)
1773 if (I->getSUnit() == FirstLocalSU)
1775 if (!DAG->
canAddEdge(FirstLocalSU, I->getSUnit()))
1782 I = LocalUses.
begin(),
E = LocalUses.
end(); I !=
E; ++
I) {
1783 DEBUG(
dbgs() <<
" Local use SU(" << (*I)->NodeNum <<
") -> SU("
1784 << GlobalSU->
NodeNum <<
")\n");
1788 I = GlobalUses.
begin(),
E = GlobalUses.
end(); I !=
E; ++
I) {
1789 DEBUG(
dbgs() <<
" Global use SU(" << (*I)->NodeNum <<
") -> SU("
1790 << FirstLocalSU->
NodeNum <<
")\n");
1802 if (FirstPos == DAG->
end())
1808 for (
unsigned Idx = 0,
End = DAG->
SUnits.size(); Idx !=
End; ++Idx) {
1813 constrainLocalCopy(SU, static_cast<ScheduleDAGMILive*>(DAG));
1836 CheckPending =
false;
1839 MinReadyCycle = UINT_MAX;
1840 ExpectedLatency = 0;
1841 DependentLatency = 0;
1843 MaxExecutedResCount = 0;
1845 IsResourceLimited =
false;
1846 ReservedCycles.
clear();
1851 MaxObservedStall = 0;
1854 ExecutedResCounts.
resize(1);
1855 assert(!ExecutedResCounts[0] &&
"nonzero count for bad resource");
1864 for (std::vector<SUnit>::iterator
1872 unsigned PIdx = PI->ProcResourceIdx;
1903 if (ReadyCycle > CurrCycle)
1904 return ReadyCycle - CurrCycle;
1912 unsigned NextUnreserved = ReservedCycles[PIdx];
1918 NextUnreserved += Cycles;
1919 return NextUnreserved;
1952 if (NRCycle > CurrCycle) {
1954 MaxObservedStall = std::max(PI->Cycles, MaxObservedStall);
1958 <<
"=" << NRCycle <<
"c\n");
1969 SUnit *LateSU =
nullptr;
1970 unsigned RemLatency = 0;
1974 if (L > RemLatency) {
1981 << LateSU->
NodeNum <<
") " << RemLatency <<
"c\n");
2000 PIdx != PEnd; ++PIdx) {
2002 if (OtherCount > OtherCritCount) {
2003 OtherCritCount = OtherCount;
2004 OtherCritIdx = PIdx;
2012 return OtherCritCount;
2022 if (ReadyCycle > CurrCycle)
2023 MaxObservedStall = std::max(ReadyCycle - CurrCycle, MaxObservedStall);
2026 if (ReadyCycle < MinReadyCycle)
2027 MinReadyCycle = ReadyCycle;
2032 if ((!IsBuffered && ReadyCycle > CurrCycle) ||
checkHazard(SU) ||
2042 assert(MinReadyCycle < UINT_MAX &&
"MinReadyCycle uninitialized");
2043 if (MinReadyCycle > NextCycle)
2044 NextCycle = MinReadyCycle;
2048 CurrMOps = (CurrMOps <= DecMOps) ? 0 : CurrMOps - DecMOps;
2051 if ((NextCycle - CurrCycle) > DependentLatency)
2052 DependentLatency = 0;
2054 DependentLatency -= (NextCycle - CurrCycle);
2058 CurrCycle = NextCycle;
2061 for (; CurrCycle != NextCycle; ++CurrCycle) {
2068 CheckPending =
true;
2078 ExecutedResCounts[PIdx] += Count;
2079 if (ExecutedResCounts[PIdx] > MaxExecutedResCount)
2080 MaxExecutedResCount = ExecutedResCounts[PIdx];
2093 unsigned Count = Factor * Cycles;
2095 <<
" +" << Cycles <<
"x" << Factor <<
"u\n");
2105 ZoneCritResIdx = PIdx;
2106 DEBUG(
dbgs() <<
" *** Critical resource "
2112 if (NextAvailable > CurrCycle) {
2115 << NextAvailable <<
"\n");
2117 return NextAvailable;
2137 "Cannot schedule this instruction's MicroOps in the current cycle.");
2140 DEBUG(
dbgs() <<
" Ready @" << ReadyCycle <<
"c\n");
2142 unsigned NextCycle = CurrCycle;
2145 assert(ReadyCycle <= CurrCycle &&
"Broken PendingQueue");
2148 if (ReadyCycle > NextCycle) {
2149 NextCycle = ReadyCycle;
2150 DEBUG(
dbgs() <<
" *** Stall until: " << ReadyCycle <<
"\n");
2159 NextCycle = ReadyCycle;
2162 RetiredMOps += IncMOps;
2169 if (ZoneCritResIdx) {
2171 unsigned ScaledMOps =
2179 DEBUG(
dbgs() <<
" *** Critical resource NumMicroOps: "
2188 if (RCycle > NextCycle)
2199 unsigned PIdx = PI->ProcResourceIdx;
2202 ReservedCycles[PIdx] =
2206 ReservedCycles[PIdx] = NextCycle;
2212 unsigned &TopLatency =
isTop() ? ExpectedLatency : DependentLatency;
2213 unsigned &BotLatency =
isTop() ? DependentLatency : ExpectedLatency;
2217 <<
" TopLatency SU(" << SU->
NodeNum <<
") " << TopLatency <<
"c\n");
2222 <<
" BotLatency SU(" << SU->
NodeNum <<
") " << BotLatency <<
"c\n");
2225 if (NextCycle > CurrCycle) {
2239 CurrMOps += IncMOps;
2241 DEBUG(
dbgs() <<
" *** Max MOps " << CurrMOps
2242 <<
" at cycle " << CurrCycle <<
'\n');
2253 MinReadyCycle = UINT_MAX;
2262 if (ReadyCycle < MinReadyCycle)
2263 MinReadyCycle = ReadyCycle;
2265 if (!IsBuffered && ReadyCycle > CurrCycle)
2278 CheckPending =
false;
2332 if (ZoneCritResIdx) {
2341 <<
" Retired: " << RetiredMOps;
2343 dbgs() <<
"\n Critical: " << ResCount / LFactor <<
"c, "
2344 << ResCount / ResFactor <<
" "
2346 <<
"\n ExpectedLatency: " << ExpectedLatency <<
"c\n"
2347 << (IsResourceLimited ?
" - Resource" :
" - Latency")
2396 RemLatency = std::max(RemLatency,
2398 RemLatency = std::max(RemLatency,
2402 unsigned OtherCritIdx = 0;
2403 unsigned OtherCount =
2406 bool OtherResLimited =
false;
2409 OtherResLimited = (int)(OtherCount - (RemLatency * LFactor)) > (
int)LFactor;
2414 if (!OtherResLimited) {
2418 <<
" RemainingLatency " << RemLatency <<
" + "
2433 if (OtherResLimited)
2434 dbgs() <<
" RemainingLimit: "
2437 dbgs() <<
" Latency limited both directions.\n");
2442 if (OtherResLimited)
2450 case NoCand:
return "NOCAND ";
2451 case Only1:
return "ONLY1 ";
2455 case Stall:
return "STALL ";
2456 case Cluster:
return "CLUSTER ";
2457 case Weak:
return "WEAK ";
2458 case RegMax:
return "REG-MAX ";
2473 unsigned ResIdx = 0;
2517 dbgs() <<
" " << Latency <<
" cycles ";
2529 if (TryVal < CandVal) {
2533 if (TryVal > CandVal) {
2534 if (Cand.
Reason > Reason)
2545 if (TryVal > CandVal) {
2549 if (TryVal < CandVal) {
2550 if (Cand.
Reason > Reason)
2583 DEBUG(
dbgs() <<
"Pick " << (IsTop ?
"Top " :
"Bot ")
2593 "(PreRA)GenericScheduler needs vreg liveness");
2607 if (!Top.HazardRec) {
2609 DAG->MF.getSubtarget().getInstrInfo()->CreateTargetMIHazardRecognizer(
2612 if (!Bot.HazardRec) {
2614 DAG->MF.getSubtarget().getInstrInfo()->CreateTargetMIHazardRecognizer(
2617 TopCand.SU =
nullptr;
2618 BotCand.SU =
nullptr;
2624 unsigned NumRegionInstrs) {
2631 RegionPolicy.ShouldTrackPressure =
true;
2637 RegionPolicy.ShouldTrackPressure = NumRegionInstrs > (NIntRegs / 2);
2643 RegionPolicy.OnlyBottomUp =
true;
2650 RegionPolicy.ShouldTrackPressure =
false;
2655 "-misched-topdown incompatible with -misched-bottomup");
2658 if (RegionPolicy.OnlyBottomUp)
2659 RegionPolicy.OnlyTopDown =
false;
2663 if (RegionPolicy.OnlyTopDown)
2664 RegionPolicy.OnlyBottomUp =
false;
2669 dbgs() <<
"GenericScheduler RegionPolicy: "
2670 <<
" ShouldTrackPressure=" << RegionPolicy.ShouldTrackPressure
2671 <<
" OnlyTopDown=" << RegionPolicy.OnlyTopDown
2672 <<
" OnlyBottomUp=" << RegionPolicy.OnlyBottomUp
2690 unsigned IterCount =
2696 unsigned InFlightCount =
2698 unsigned BufferLimit =
2706 <<
"c NumIters=" << (AcyclicCount + IterCount-1) / IterCount
2710 dbgs() <<
" ACYCLIC LATENCY LIMIT\n");
2717 for (std::vector<SUnit*>::const_iterator
2718 I = Bot.Available.begin(),
E = Bot.Available.end(); I !=
E; ++
I) {
2729 checkAcyclicLatency();
2748 if (Cand.AtTop != TryCand.
AtTop)
2755 if (TryPSet == CandPSet) {
2761 std::numeric_limits<int>::max();
2764 std::numeric_limits<int>::max();
2769 return tryGreater(TryRank, CandRank, TryCand, Cand, Reason);
2788 unsigned ScheduledOper = isTop ? 1 : 0;
2789 unsigned UnscheduledOper = isTop ? 0 : 1;
2800 return AtBoundary ? -1 : 1;
2810 if (DAG->isTrackingPressure()) {
2815 DAG->getRegionCriticalPSets(),
2816 DAG->getRegPressure().MaxSetPressure);
2821 &DAG->getPressureDiff(Cand.
SU),
2823 DAG->getRegionCriticalPSets(),
2824 DAG->getRegPressure().MaxSetPressure);
2828 DAG->getPressureDiff(Cand.
SU),
2830 DAG->getRegionCriticalPSets(),
2831 DAG->getRegPressure().MaxSetPressure);
2884 bool SameBoundary = Zone !=
nullptr;
2905 const SUnit *CandNextClusterSU =
2907 const SUnit *TryCandNextClusterSU =
2910 Cand.
SU == CandNextClusterSU,
2918 TryCand, Cand,
Weak))
2970 initCandidate(TryCand, *I, Zone.
isTop(), RPTracker, TempTracker);
2973 tryCandidate(Cand, TryCand, ZoneArg);
2988 if (
SUnit *SU = Bot.pickOnlyChoice()) {
2993 if (
SUnit *SU = Top.pickOnlyChoice()) {
3009 if (!BotCand.isValid() || BotCand.SU->isScheduled ||
3010 BotCand.Policy != BotPolicy) {
3012 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand);
3013 assert(BotCand.Reason !=
NoCand &&
"failed to find the first candidate");
3020 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), TCand);
3022 "Last pick result should correspond to re-picking right now");
3029 if (!TopCand.isValid() || TopCand.SU->isScheduled ||
3030 TopCand.Policy != TopPolicy) {
3032 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand);
3033 assert(TopCand.Reason !=
NoCand &&
"failed to find the first candidate");
3040 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TCand);
3042 "Last pick result should correspond to re-picking right now");
3048 assert(BotCand.isValid());
3049 assert(TopCand.isValid());
3052 tryCandidate(Cand, TopCand,
nullptr);
3053 if (TopCand.Reason !=
NoCand) {
3054 Cand.setBest(TopCand);
3058 IsTopNode = Cand.AtTop;
3066 assert(Top.Available.empty() && Top.Pending.empty() &&
3067 Bot.Available.empty() && Bot.Pending.empty() &&
"ReadyQ garbage");
3072 if (RegionPolicy.OnlyTopDown) {
3073 SU = Top.pickOnlyChoice();
3076 TopCand.reset(NoPolicy);
3077 pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand);
3078 assert(TopCand.Reason !=
NoCand &&
"failed to find a candidate");
3083 }
else if (RegionPolicy.OnlyBottomUp) {
3084 SU = Bot.pickOnlyChoice();
3087 BotCand.reset(NoPolicy);
3088 pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand);
3089 assert(BotCand.Reason !=
NoCand &&
"failed to find a candidate");
3095 SU = pickNodeBidirectional(IsTopNode);
3100 Top.removeReady(SU);
3102 Bot.removeReady(SU);
3121 SUnit *DepSU = I->getSUnit();
3122 if (isTop ? DepSU->
Succs.size() > 1 : DepSU->
Preds.size() > 1)
3127 DEBUG(
dbgs() <<
" Rescheduling physreg copy ";
3128 I->getSUnit()->dump(DAG));
3145 reschedulePhysRegCopies(SU,
true);
3150 reschedulePhysRegCopies(SU,
false);
3191 if (!Top.HazardRec) {
3204 I = BotRoots.begin(),
E = BotRoots.end(); I !=
E; ++
I) {
3228 if (
tryLess(Top.getLatencyStallCycles(TryCand.
SU),
3229 Top.getLatencyStallCycles(Cand.
SU), TryCand, Cand,
Stall))
3237 Cand.ResDelta.DemandedResources,
3242 if (Cand.Policy.ReduceLatency &&
tryLatency(TryCand, Cand, Top)) {
3247 if (TryCand.
SU->
NodeNum < Cand.SU->NodeNum)
3256 TryCand.
AtTop =
true;
3258 tryCandidate(Cand, TryCand);
3269 assert(Top.Available.empty() && Top.Pending.empty() &&
"ReadyQ garbage");
3274 SU = Top.pickOnlyChoice();
3283 pickNodeFromQueue(TopCand);
3291 Top.removeReady(SU);
3305 return new ScheduleDAGMI(C, make_unique<PostGenericScheduler>(C),
3320 ILPOrder(
bool MaxILP)
3321 : DFSResult(nullptr), ScheduledTrees(nullptr), MaximizeILP(MaxILP) {}
3326 bool operator()(
const SUnit *
A,
const SUnit *
B)
const {
3327 unsigned SchedTreeA = DFSResult->getSubtreeID(A);
3328 unsigned SchedTreeB = DFSResult->getSubtreeID(B);
3329 if (SchedTreeA != SchedTreeB) {
3331 if (ScheduledTrees->test(SchedTreeA) != ScheduledTrees->test(SchedTreeB))
3332 return ScheduledTrees->test(SchedTreeB);
3335 if (DFSResult->getSubtreeLevel(SchedTreeA)
3336 != DFSResult->getSubtreeLevel(SchedTreeB)) {
3337 return DFSResult->getSubtreeLevel(SchedTreeA)
3338 < DFSResult->getSubtreeLevel(SchedTreeB);
3342 return DFSResult->getILP(A) < DFSResult->getILP(B);
3344 return DFSResult->getILP(A) > DFSResult->getILP(B);
3353 std::vector<SUnit*> ReadyQ;
3355 ILPScheduler(
bool MaximizeILP): DAG(nullptr), Cmp(MaximizeILP) {}
3361 Cmp.DFSResult = DAG->getDFSResult();
3362 Cmp.ScheduledTrees = &DAG->getScheduledTrees();
3368 std::make_heap(ReadyQ.begin(), ReadyQ.end(), Cmp);
3376 if (ReadyQ.empty())
return nullptr;
3377 std::pop_heap(ReadyQ.begin(), ReadyQ.end(), Cmp);
3378 SUnit *SU = ReadyQ.back();
3381 DEBUG(
dbgs() <<
"Pick node " <<
"SU(" << SU->NodeNum <<
") "
3382 <<
" ILP: " << DAG->getDFSResult()->getILP(SU)
3383 <<
" Tree: " << DAG->getDFSResult()->getSubtreeID(SU) <<
" @"
3384 << DAG->getDFSResult()->getSubtreeLevel(
3385 DAG->getDFSResult()->getSubtreeID(SU)) <<
'\n'
3386 <<
"Scheduling " << *SU->getInstr());
3392 std::make_heap(ReadyQ.begin(), ReadyQ.end(), Cmp);
3398 assert(!IsTopNode &&
"SchedDFSResult needs bottom-up");
3404 ReadyQ.push_back(SU);
3405 std::push_heap(ReadyQ.begin(), ReadyQ.end(), Cmp);
3429 template<
bool IsReverse>
3453 InstructionShuffler(
bool alternate,
bool topdown)
3454 : IsAlternating(alternate), IsTopDown(topdown) {}
3468 if (TopQ.empty())
return nullptr;
3475 if (BottomQ.empty())
return nullptr;
3482 IsTopDown = !IsTopDown;
3501 "-misched-topdown incompatible with -misched-bottomup");
3502 return new ScheduleDAGMILive(C, make_unique<InstructionShuffler>(Alternate, TopDown));
3505 "shuffle",
"Shuffle machine instructions alternating directions",
3545 return "color=cyan,style=dashed";
3547 return "color=blue,style=dashed";
3567 std::string Str(
"shape=Mrecord");
3572 Str +=
",style=filled,fillcolor=\"#";
3589 errs() <<
"ScheduleDAGMI::viewGraph is only available in debug builds on "
3590 <<
"systems with Graphviz or gv!\n";
3596 viewGraph(getDAGName(),
"Scheduling-Units Graph for " + getDAGName());
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
iterator end()
Returns an iterator past this container.
virtual ScheduleHazardRecognizer * CreateTargetMIHazardRecognizer(const InstrItineraryData *, const ScheduleDAG *DAG) const
Allocate and return a hazard recognizer to use for this target when scheduling the machine instructio...
VReg2SUnitMultiMap VRegUses
Maps vregs to the SUnits of their uses in the current scheduling region.
void computeDFSResult()
Compute a DFSResult after DAG building is complete, and before any queue comparisons.
void releaseSucc(SUnit *SU, SDep *SuccEdge)
ReleaseSucc - Decrement the NumPredsLeft count of a successor.
static int biasPhysRegCopy(const SUnit *SU, bool isTop)
Minimize physical register live ranges.
Weak DAG edge linking a chain of clustered instrs.
void push_back(const T &Elt)
void schedNode(SUnit *SU, bool IsTopNode) override
Called after ScheduleDAGMI has scheduled an instruction and updated scheduled/remaining flags in the ...
const_iterator end(StringRef path)
Get end iterator over path.
virtual void finishBlock()
finishBlock - Clean up after scheduling in the given block.
virtual void initialize(ScheduleDAGMI *DAG)=0
Initialize the strategy after building the DAG for a new region.
bool verify(Pass *p=nullptr, const char *Banner=nullptr, bool AbortOnError=true) const
Run the current MachineFunction through the machine code verifier, useful for debugger use...
virtual bool enableMachineScheduler() const
True if the subtarget should run MachineScheduler after aggressive coalescing.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static cl::opt< bool > ViewMISchedDAGs("view-misched-dags", cl::Hidden, cl::desc("Pop up a window to show MISched dags after they are processed"))
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static std::string getNodeAttributes(const SUnit *N, const ScheduleDAG *G)
static cl::opt< bool > EnablePostRAMachineSched("enable-post-misched", cl::desc("Enable the post-ra machine instruction scheduling pass."), cl::init(true), cl::Hidden)
void clear()
Clear the results.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Segments::iterator iterator
bool isArtificialDep() const
void detectDeadDefs(const MachineInstr &MI, const LiveIntervals &LIS)
Use liveness information to find dead defs not marked with a dead flag and move them to the DeadDefs ...
virtual ~MachineSchedContext()
Each Scheduling boundary is associated with ready queues.
SlotIndex def
The index of the defining instruction.
iterator insert(const ValueT &Val)
Insert a new element at the tail of the subset list.
void addPressureChange(unsigned RegUnit, bool IsDec, const MachineRegisterInfo *MRI)
Add a change in pressure to the pressure diff of a given instruction.
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
const MachineSchedContext * Context
SUnit * pickNodeBidirectional(bool &IsTopNode)
Pick the best candidate node from either the top or bottom queue.
static bool tryPressure(const PressureChange &TryP, const PressureChange &CandP, GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, GenericSchedulerBase::CandReason Reason, const TargetRegisterInfo *TRI, const MachineFunction &MF)
virtual void releaseTopNode(SUnit *SU)=0
When all predecessor dependencies have been resolved, free this node for top-down scheduling...
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
void init(ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel)
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
A Module instance is used to store all the information related to an LLVM module. ...
bool isInQueue(SUnit *SU) const
unsigned getCurrCycle() const
Number of cycles to issue the instructions scheduled in this zone.
static MachineSchedRegistry ILPMaxRegistry("ilpmax","Schedule bottom-up for max ILP", createILPMaxScheduler)
ScheduleDAGTopologicalSort Topo
Topo - A topological ordering for SUnits which permits fast IsReachable and similar queries...
static bool tryGreater(int TryVal, int CandVal, GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, GenericSchedulerBase::CandReason Reason)
MachineBasicBlock::iterator CurrentTop
The top of the unscheduled zone.
bool TrackLaneMasks
Whether lane masks should get tracked.
char & MachineDominatorsID
MachineDominators - This pass is a machine dominators analysis pass.
LiveInterval - This class represents the liveness of a register, or stack slot.
static cl::opt< std::string > SchedOnlyFunc("misched-only-func", cl::Hidden, cl::desc("Only schedule this function"))
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
MachineInstr * getInstr() const
getInstr - Return the representative MachineInstr for this SUnit.
unsigned computeCyclicCriticalPath()
Compute the cyclic critical path through the DAG.
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
virtual void overrideSchedPolicy(MachineSchedPolicy &Policy, unsigned NumRegionInstrs) const
Override generic scheduling policy within a region.
SlotIndex getMBBEndIdx(const MachineBasicBlock *mbb) const
Return the last index in the given basic block.
void traceCandidate(const SchedCandidate &Cand)
unsigned DemandedResources
static bool renderGraphFromBottomUp()
bool ShouldTrackPressure
Register pressure in this region computed by initRegPressure.
bool isLocal(SlotIndex Start, SlotIndex End) const
True iff this segment is a single segment that lies between the specified boundaries, exclusively.
MachineBasicBlock::iterator begin() const
begin - Return an iterator to the top of the current scheduling region.
ScheduleDAGMI * createGenericSchedPostRA(MachineSchedContext *C)
Create a generic scheduler with no vreg liveness or DAG mutation passes.
unsigned getMicroOpBufferSize() const
Number of micro-ops that may be buffered for OOO execution.
Mutate the DAG as a postpass after normal DAG building.
const_iterator begin(StringRef path)
Get begin iterator over path.
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
static ScheduleDAGInstrs * createILPMaxScheduler(MachineSchedContext *C)
virtual std::string getGraphNodeLabel(const SUnit *SU) const =0
getGraphNodeLabel - Return a label for an SUnit node in a visualization of the ScheduleDAG.
const MCSchedClassDesc * getSchedClass(SUnit *SU) const
Resolve and cache a resolved scheduling class for an SUnit.
static ScheduleDAGInstrs * createInstructionShuffler(MachineSchedContext *C)
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
char & MachineSchedulerID
MachineScheduler - This pass schedules machine instructions.
void closeRegion()
Finalize the region boundaries and recored live ins and live outs.
Summarize the unscheduled region.
virtual bool enablePostRAScheduler() const
True if the subtarget should run a scheduler after register allocation.
void reset(const CandPolicy &NewPolicy)
unsigned getResourceFactor(unsigned ResIdx) const
Multiply the number of units consumed for a resource by this factor to normalize it relative to other...
unsigned getNumAllocatableRegs(const TargetRegisterClass *RC) const
getNumAllocatableRegs - Returns the number of actually allocatable registers in RC in the current fun...
RegisterPassParser class - Handle the addition of new machine passes.
MachineSchedRegistry provides a selection of available machine instruction schedulers.
RegisterClassInfo * RegClassInfo
MachineBasicBlock::iterator top() const
VNInfo - Value Number Information.
iterator_range< mop_iterator > operands()
static std::string getNodeLabel(const SUnit *SU, const ScheduleDAG *G)
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
std::unique_ptr< MachineSchedStrategy > SchedImpl
virtual bool shouldClusterMemOps(MachineInstr &FirstLdSt, MachineInstr &SecondLdSt, unsigned NumLoads) const
Returns true if the two given memory operations should be scheduled adjacent.
void fixupKills(MachineBasicBlock *MBB)
Fix register kill flags that scheduling has made invalid.
bool isArtificial() const
isArtificial - Test if this is an Order dependence that is marked as "artificial", meaning it isn't necessary for correctness.
nonconst_iterator getNonConstIterator() const
INITIALIZE_PASS_BEGIN(MachineScheduler,"machine-scheduler","Machine Instruction Scheduler", false, false) INITIALIZE_PASS_END(MachineScheduler
virtual void AdvanceCycle()
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
void updateQueues(SUnit *SU, bool IsTopNode)
Update scheduler DAG and queues after scheduling an instruction.
virtual void schedNode(SUnit *SU, bool IsTopNode)=0
Notify MachineSchedStrategy that ScheduleDAGMI has scheduled an instruction and updated scheduled/rem...
SmallVector< SDep, 4 > Preds
virtual void startBlock(MachineBasicBlock *BB)
startBlock - Prepare to perform scheduling in the given block.
MachineBasicBlock::const_iterator getPos() const
Get the MI position corresponding to this register pressure.
A register anti-dependedence (aka WAR).
ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules machine instructions while...
iterator_base< SparseMultiSet * > iterator
void dumpAll(const ScheduleDAG *G) const
unsigned getDependentLatency() const
void recede(SmallVectorImpl< RegisterMaskPair > *LiveUses=nullptr)
Recede across the previous instruction.
const TargetSchedModel * getSchedModel() const
Get the machine model for instruction scheduling.
unsigned NumInstrsScheduled
The number of instructions scheduled so far.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
unsigned getNumSubtrees() const
The number of subtrees detected in this DAG.
unsigned getHeight() const
getHeight - Return the height of this node, which is the length of the maximum path down to any node ...
ArrayRef< SUnit * > elements()
void buildSchedGraph(AliasAnalysis *AA, RegPressureTracker *RPTracker=nullptr, PressureDiffs *PDiffs=nullptr, LiveIntervals *LIS=nullptr, bool TrackLaneMasks=false)
buildSchedGraph - Build SUnits from the MachineBasicBlock that we are input.
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
void clear()
clear - Clear all bits.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineBasicBlock::iterator RegionEnd
The end of the range to be scheduled.
void closeBottom()
Set the boundary for the bottom of the region and summarize live outs.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
bool isWeak() const
isWeak - Test if this a weak dependence.
Provide an instruction scheduling machine model to CodeGen passes.
const HexagonInstrInfo * TII
static std::string getEdgeAttributes(const SUnit *Node, SUnitIterator EI, const ScheduleDAG *Graph)
If you want to override the dot attributes printed for a particular edge, override this method...
An individual mapping from virtual register number to SUnit.
static MachineSchedRegistry DefaultSchedRegistry("default","Use the target's default scheduler choice.", useDefaultMachineSched)
static cl::opt< MachineSchedRegistry::ScheduleDAGCtor, false, RegisterPassParser< MachineSchedRegistry > > MachineSchedOpt("misched", cl::init(&useDefaultMachineSched), cl::Hidden, cl::desc("Machine instruction scheduler to use"))
MachineSchedOpt allows command line selection of the scheduler.
virtual const char * getRegPressureSetName(unsigned Idx) const =0
Get the name of this register unit pressure set.
static unsigned getWeakLeft(const SUnit *SU, bool isTop)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Regular data dependence (aka true-dependence).
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
Result of a LiveRange query.
Reg
All possible values of the reg field in the ModR/M byte.
PressureDiff & getPressureDiff(const SUnit *SU)
std::vector< std::unique_ptr< ScheduleDAGMutation > > Mutations
Ordered list of DAG postprocessing steps.
void InitDAGTopologicalSorting()
InitDAGTopologicalSorting - create the initial topological ordering from the DAG to be scheduled...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Target-Independent Code Generator Pass Configuration Options.
bool IsAcyclicLatencyLimited
static MachineSchedRegistry ShufflerRegistry("shuffle","Shuffle machine instructions alternating directions", createInstructionShuffler)
void initPolicy(MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, unsigned NumRegionInstrs) override
Initialize the per-region scheduling policy.
static bool isSimple(Instruction *I)
bool ShouldTrackLaneMasks
Compute the values of each DAG node for various metrics during DFS.
void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs) override
Implement the ScheduleDAGInstrs interface for handling the next scheduling region.
static const unsigned InvalidCycle
MachineBasicBlock::iterator LiveRegionEnd
unsigned getCriticalCount() const
Get the scaled count of scheduled micro-ops and resources, including executed resources.
static cl::opt< bool > VerifyScheduling("verify-misched", cl::Hidden, cl::desc("Verify machine instrs before and after machine scheduling"))
void bumpCycle(unsigned NextCycle)
Move the boundary of scheduled code by one cycle.
unsigned countResource(unsigned PIdx, unsigned Cycles, unsigned ReadyCycle)
Add the given processor resource to this scheduled zone.
static MachineBasicBlock::const_iterator priorNonDebug(MachineBasicBlock::const_iterator I, MachineBasicBlock::const_iterator Beg)
Decrement this iterator until reaching the top or a non-debug instr.
const_iterator end() const
void dumpPolicy() override
void scheduleMI(SUnit *SU, bool IsTopNode)
Move an instruction and update register pressure.
MachineBasicBlock::iterator RegionBegin
The beginning of the range to be scheduled.
reverse_iterator rbegin() const
RegPressureTracker BotRPTracker
void buildDAGWithRegPressure()
Call ScheduleDAGInstrs::buildSchedGraph with register pressure tracking enabled.
virtual unsigned getRegPressureSetScore(const MachineFunction &MF, unsigned PSetID) const
Return a heuristic for the machine scheduler to compare the profitability of increasing one register ...
virtual void RecedeCycle()
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
SmallVector< RegisterMaskPair, 8 > LiveInRegs
List of live in virtual registers or physical register units.
std::vector< PressureChange > RegionCriticalPSets
List of pressure sets that exceed the target's pressure limit before scheduling, listed in increasing...
static const char * getReasonStr(GenericSchedulerBase::CandReason Reason)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
static std::string getGraphName(const ScheduleDAG *G)
Itinerary data supplied by a subtarget to be used by a target.
static void tracePick(GenericSchedulerBase::CandReason Reason, bool IsTop)
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
void releasePending()
Release pending ready nodes in to the available queue.
void collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, bool TrackLaneMasks, bool IgnoreDead)
Analyze the given instruction MI and fill in the Uses, Defs and DeadDefs list based on the MachineOpe...
COFF::MachineTypes Machine
ScheduleDAGMILive * createGenericSchedLive(MachineSchedContext *C)
Create the standard converging machine scheduler.
MachinePassRegistry - Track the registration of machine passes.
List of registers defined and used by a machine instruction.
std::unique_ptr< ScheduleDAGMutation > createMacroFusionDAGMutation(const TargetInstrInfo *TII)
void collectVRegUses(SUnit &SU)
virtual void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs)
Initialize the scheduler state for the next scheduling region.
void bumpNode(SUnit *SU)
Move the boundary of scheduled code by one SUnit.
Printable PrintReg(unsigned Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubRegIdx=0)
Prints virtual and physical registers with or without a TRI instance.
SUnit * pickNode(bool &IsTopNode) override
Pick the next node to schedule.
size_t size() const
size - Get the array size.
void initCandidate(SchedCandidate &Cand, SUnit *SU, bool AtTop, const RegPressureTracker &RPTracker, RegPressureTracker &TempTracker)
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
void checkAcyclicLatency()
Set IsAcyclicLatencyLimited if the acyclic path is longer than the cyclic critical path by more cycle...
static std::string getNodeDescription(const SUnit *SU, const ScheduleDAG *G)
void incExecutedResources(unsigned PIdx, unsigned Count)
TargetInstrInfo - Interface to description of machine instruction set.
virtual void registerRoots()
Notify this strategy that all roots have been released (including those that depend on EntrySU or Exi...
bool isDebugValue() const
void resize(unsigned NumSUnits)
Initialize the result data with the size of the DAG.
std::vector< SUnit * >::iterator iterator
void recedeSkipDebugValues()
Recede until we find an instruction which is not a DebugValue.
SDep - Scheduling dependency.
SmallVector< RegisterMaskPair, 8 > LiveOutRegs
void addLiveRegs(ArrayRef< RegisterMaskPair > Regs)
Force liveness of virtual registers or physical register units.
reverse_iterator rend() const
RegisterClassInfo * RegClassInfo
void getMaxDownwardPressureDelta(const MachineInstr *MI, RegPressureDelta &Delta, ArrayRef< PressureChange > CriticalPSets, ArrayRef< unsigned > MaxPressureLimit)
Consider the pressure increase caused by traversing this instruction top-down.
iterator remove(iterator I)
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
initializer< Ty > init(const Ty &Val)
ScheduleHazardRecognizer * HazardRec
unsigned getNextResourceCycle(unsigned PIdx, unsigned Cycles)
Compute the next cycle at which the given processor resource can be scheduled.
void initQueues(ArrayRef< SUnit * > TopRoots, ArrayRef< SUnit * > BotRoots)
Release ExitSU predecessors and setup scheduler queues.
LiveQueryResult Query(SlotIndex Idx) const
Query Liveness at Idx.
CandReason
Represent the type of SchedCandidate found within a single queue.
ArrayRef< unsigned > getLiveThru() const
Helpers for implementing custom MachineSchedStrategy classes.
PressureChange CurrentMax
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
unsigned getScheduledLatency() const
Get the number of latency cycles "covered" by the scheduled instructions.
const InstrItineraryData * getInstrItineraries() const
static const unsigned MinSubtreeSize
Identify one of the processor resource kinds consumed by a particular scheduling class for the specif...
void dumpSchedule() const
dump the scheduled Sequence.
unsigned getZoneCritResIdx() const
void removeReady(SUnit *SU)
Remove SU from the ready set for this boundary.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
const MachineOperand & getOperand(unsigned i) const
Summarize the scheduling resources required for an instruction of a particular scheduling class...
void updatePressureDiffs(ArrayRef< RegisterMaskPair > LiveUses)
Update the PressureDiff array for liveness after scheduling this instruction.
unsigned getLatencyFactor() const
Multiply cycle count by this factor to normalize it relative to other resources.
static bool isSchedBoundary(MachineBasicBlock::iterator MI, MachineBasicBlock *MBB, MachineFunction *MF, const TargetInstrInfo *TII)
Return true of the given instruction should not be included in a scheduling region.
RegisterPressure computed within a region of instructions delimited by TopPos and BottomPos...
bool addEdge(SUnit *SuccSU, const SDep &PredDep)
Add a DAG edge to the given SU with the given predecessor dependence data.
void setUniverse(unsigned U)
Set the universe size which determines the largest key the set can hold.
Represent the analysis usage information of a pass.
static MachinePassRegistry Registry
unsigned getLatency() const
getLatency - Return the latency value for this edge, which roughly means the minimum number of cycles...
Store the state used by GenericScheduler heuristics, required for the lifetime of one invocation of p...
bool checkHazard(SUnit *SU)
Does this SU have a hazard within the current instruction group.
void tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand, SchedBoundary *Zone)
Apply a set of heursitics to a new candidate.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
void releasePred(SUnit *SU, SDep *PredEdge)
ReleasePred - Decrement the NumSuccsLeft count of a predecessor.
Track the current register pressure at some position in the instruction stream, and remember the high...
static const unsigned End
virtual void releaseBottomNode(SUnit *SU)=0
When all successor dependencies have been resolved, free this node for bottom-up scheduling.
void findRootsAndBiasEdges(SmallVectorImpl< SUnit * > &TopRoots, SmallVectorImpl< SUnit * > &BotRoots)
Policy for scheduling the next instruction in the candidate's zone.
const TargetSchedModel * SchedModel
static ScheduleDAGInstrs * useDefaultMachineSched(MachineSchedContext *C)
A dummy default scheduler factory indicates whether the scheduler is overridden on the command line...
List of PressureChanges in order of increasing, unique PSetID.
virtual void exitRegion()
Notify that the scheduler has finished scheduling the current region.
unsigned getRegPressureSetLimit(unsigned Idx) const
Get the register unit limit for the given pressure set index.
void ViewGraph(const GraphType &G, const Twine &Name, bool ShortNames=false, const Twine &Title="", GraphProgram::Name Program=GraphProgram::DOT)
ViewGraph - Emit a dot graph, run 'dot', run gv on the postscript file, then cleanup.
DOTGraphTraits(bool isSimple=false)
bool empty() const
empty - Check if the array is empty.
SchedResourceDelta ResDelta
const TargetRegisterInfo * TRI
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
std::unique_ptr< ScheduleDAGMutation > createStoreClusterDAGMutation(const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
SchedDFSResult * DFSResult
Information about DAG subtrees.
PressureDiffs SUPressureDiffs
void dumpScheduledState()
void clear()
Clears the set.
unsigned getNumProcResourceKinds() const
Get the number of kinds of resources for this target.
static MachineBasicBlock::const_iterator nextIfDebug(MachineBasicBlock::const_iterator I, MachineBasicBlock::const_iterator End)
If this iterator is a debug value, increment until reaching the End or a non-debug instruction...
static cl::opt< bool > EnableMacroFusion("misched-fusion", cl::Hidden, cl::desc("Enable scheduling for macro fusion."), cl::init(true))
void apply(Opt *O, const Mod &M, const Mods &...Ms)
LiveIntervals * getLIS() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool isPHIDef() const
Returns true if this value is defined by a PHI instruction (or was, PHI instructions may have been el...
void releaseSuccessors(SUnit *SU)
releaseSuccessors - Call releaseSucc on each of SU's successors.
const SUnit * NextClusterSucc
std::string & str()
Flushes the stream contents to the target string and returns the string's reference.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void viewGraph() override
Out-of-line implementation with no arguments is handy for gdb.
void setPolicy(CandPolicy &Policy, bool IsPostRA, SchedBoundary &CurrZone, SchedBoundary *OtherZone)
Set the CandPolicy given a scheduling zone given the current resources and latencies inside and outsi...
void initLiveThru(const RegPressureTracker &RPTracker)
Initialize the LiveThru pressure set based on the untied defs found in RPTracker. ...
static cl::opt< bool > EnableRegPressure("misched-regpressure", cl::Hidden, cl::desc("Enable register pressure scheduling."), cl::init(true))
ProcResIter getWriteProcResEnd(const MCSchedClassDesc *SC) const
iterator find(const KeyT &Key)
Find an element by its key.
iterator find(SlotIndex Pos)
find - Return an iterator pointing to the first segment that ends after Pos, or end().
DOTGraphTraits - Template class that can be specialized to customize how graphs are converted to 'dot...
void init(ScheduleDAGMI *dag, const TargetSchedModel *smodel, SchedRemainder *rem)
bool hasInstrSchedModel() const
Return true if this machine model includes an instruction-level scheduling model. ...
static cl::opt< unsigned > SchedOnlyBlock("misched-only-block", cl::Hidden, cl::desc("Only schedule this MBB#"))
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
std::vector< unsigned > MaxSetPressure
Map of max reg pressure indexed by pressure set ID, not class ID.
void adjustLaneLiveness(const LiveIntervals &LIS, const MachineRegisterInfo &MRI, SlotIndex Pos, MachineInstr *AddFlagsMI=nullptr)
Use liveness information to find out which uses/defs are partially undefined/dead and adjust the Regi...
void advance()
Advance across the current instruction.
Iterator for intrusive lists based on ilist_node.
std::unique_ptr< ScheduleDAGMutation > createLoadClusterDAGMutation(const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
virtual void EmitInstruction(SUnit *)
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
const SUnit * NextClusterPred
Record the next node in a scheduled cluster.
const MCProcResourceDesc * getProcResource(unsigned PIdx) const
Get a processor resource by ID for convenience.
static bool isSameInstr(SlotIndex A, SlotIndex B)
isSameInstr - Return true if A and B refer to the same instruction.
void compute(ArrayRef< SUnit > SUnits)
Compute various metrics for the DAG with given roots.
void registerRoots() override
Notify this strategy that all roots have been released (including those that depend on EntrySU or Exi...
void reschedulePhysRegCopies(SUnit *SU, bool isTop)
unsigned getExecutedCount() const
Get a scaled count for the minimum execution time of the scheduled micro-ops that are ready to execut...
AnalysisUsage & addRequiredID(const void *ID)
ScheduleDAGInstrs *(* ScheduleDAGCtor)(MachineSchedContext *)
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
static cl::opt< unsigned > MISchedCutoff("misched-cutoff", cl::Hidden, cl::desc("Stop scheduling after N instructions"), cl::init(~0U))
static ScheduleDAGInstrs * createILPMinScheduler(MachineSchedContext *C)
bool test(unsigned Idx) const
bool isCtrlDep() const
isCtrlDep - Test if this is not an SDep::Data dependence.
virtual const TargetLowering * getTargetLowering() const
static cl::opt< bool > EnableMachineSched("enable-misched", cl::desc("Enable the machine instruction scheduling pass."), cl::init(true), cl::Hidden)
static LaneBitmask getNone()
std::reverse_iterator< const_iterator > const_reverse_iterator
void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs) override
Implement the ScheduleDAGInstrs interface for handling the next scheduling region.
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
cl::opt< bool > DumpCriticalPathLength("misched-dcpl", cl::Hidden, cl::desc("Print critical path length to stdout"))
void releaseNode(SUnit *SU, unsigned ReadyCycle)
CHAIN = SC CHAIN, Imm128 - System call.
StringRef getColorString(unsigned NodeNumber)
Get a color string for this node number.
void updateScheduledPressure(const SUnit *SU, const std::vector< unsigned > &NewMaxPressure)
void setPreservesCFG()
This function should be called by the pass, iff they do not:
LiveInterval & getInterval(unsigned Reg)
std::unique_ptr< ScheduleDAGMutation > createCopyConstrainDAGMutation(const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
static cl::opt< bool > EnableCyclicPath("misched-cyclicpath", cl::Hidden, cl::desc("Enable cyclic critical path analysis."), cl::init(true))
void initResourceDelta(const ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
~ScheduleDAGMILive() override
void initializeMachineSchedulerPass(PassRegistry &)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
unsigned getOtherResourceCount(unsigned &OtherCritIdx)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static bool tryLess(int TryVal, int CandVal, GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, GenericSchedulerBase::CandReason Reason)
Return true if this heuristic determines order.
virtual void finalizeSchedule()
finalizeSchedule - Allow targets to perform final scheduling actions at the level of the whole Machin...
void tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand)
Apply a set of heursitics to a new candidate for PostRA scheduling.
bool IsReachable(const SUnit *SU, const SUnit *TargetSU)
IsReachable - Checks if SU is reachable from TargetSU.
static MachineSchedRegistry GenericSchedRegistry("converge","Standard converging scheduler.", createConveringSched)
StringRef getName() const
Return the name of the corresponding LLVM basic block, or "(null)".
void biasCriticalPath()
Order this node's predecessor edges such that the critical path edge occurs first.
PressureChange CriticalMax
virtual void Reset()
Reset - This callback is invoked when a new block of instructions is about to be schedule.
virtual void schedule()=0
schedule - Order nodes according to selected style, filling in the Sequence member.
virtual bool getMemOpBaseRegImmOfs(MachineInstr &MemOp, unsigned &BaseReg, int64_t &Offset, const TargetRegisterInfo *TRI) const
Get the base register and byte offset of an instruction that reads/writes memory. ...
machine Machine Instruction Scheduler
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
MachineBasicBlock::iterator bottom() const
void moveInstruction(MachineInstr *MI, MachineBasicBlock::iterator InsertPos)
Change the position of an instruction within the basic block and update live ranges and region bounda...
MachineBasicBlock::iterator end() const
end - Return an iterator to the bottom of the current scheduling region.
virtual bool hasVRegLiveness() const
Return true if this DAG supports VReg liveness and RegPressure.
bool isBoundaryNode() const
Boundary nodes are placeholders for the boundary of the scheduling region.
unsigned getDepth() const
getDepth - Return the depth of this node, which is the length of the maximum path up to any node whic...
void schedNode(SUnit *SU, bool IsTopNode) override
Update the scheduler's state after scheduling a node.
const SUnit * getNextClusterSucc() const
virtual void scheduleTree(unsigned SubtreeID)
Scheduler callback to notify that a new subtree is scheduled.
void setLatency(unsigned Lat)
setLatency - Set the latency for this edge.
void closeTop()
Set the boundary for the top of the region and summarize live ins.
void releasePredecessors(SUnit *SU)
releasePredecessors - Call releasePred on each of SU's predecessors.
unsigned getSubtreeID(const SUnit *SU) const
Get the ID of the subtree the given DAG node belongs to.
unsigned getUnscheduledLatency(SUnit *SU) const
void initialize(ScheduleDAGMI *dag) override
Initialize the strategy after building the DAG for a new region.
SlotIndex beginIndex() const
beginIndex - Return the lowest numbered slot covered.
bool isCluster() const
isCluster - Test if this is an Order dependence that is marked as "cluster", meaning it is artificial...
SUnit * getSUnit(MachineInstr *MI) const
getSUnit - Return an existing SUnit for this MI, or NULL.
virtual SUnit * pickNode(bool &IsTopNode)=0
Pick the next node to schedule, or return NULL.
unsigned getNumMicroOps(const MachineInstr *MI, const MCSchedClassDesc *SC=nullptr) const
Return the number of issue slots required for this MI.
ScheduleDAGInstrs - A ScheduleDAG subclass for scheduling lists of MachineInstrs. ...
Representation of each machine instruction.
void initializePostMachineSchedulerPass(PassRegistry &)
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
machine Machine Instruction false
void initialize(ScheduleDAGMI *Dag) override
Initialize the strategy after building the DAG for a new region.
const TargetRegisterInfo * TRI
virtual bool shouldScheduleAdjacent(const MachineInstr &First, const MachineInstr &Second) const
Can this target fuse the given instructions if they are scheduled adjacent.
SUnit * pickOnlyChoice()
Call this before applying any other heuristics to the Available queue.
void dumpRegSetPressure(ArrayRef< unsigned > SetPressure, const TargetRegisterInfo *TRI)
Status of an instruction's critical resource consumption.
~ScheduleDAGMI() override
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
char & PostMachineSchedulerID
PostMachineScheduler - This pass schedules machine instructions postRA.
cl::opt< bool > ForceBottomUp
LLVM_DUMP_METHOD void dump(const TargetRegisterInfo &TRI) const
unsigned findMaxLatency(ArrayRef< SUnit * > ReadySUs)
SlotIndex endIndex() const
endNumber - return the maximum point of the range of the whole, exclusive.
unsigned getNumInstrs(const SUnit *SU) const
Get the number of instructions in the given subtree and its children.
MachineSchedContext provides enough context from the MachineScheduler pass for the target to instanti...
MachineSchedStrategy - Interface to the scheduling algorithm used by ScheduleDAGMI.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Capture a change in pressure for a single pressure set.
void getMaxUpwardPressureDelta(const MachineInstr *MI, PressureDiff *PDiff, RegPressureDelta &Delta, ArrayRef< PressureChange > CriticalPSets, ArrayRef< unsigned > MaxPressureLimit)
Consider the pressure increase caused by traversing this instruction bottom-up.
SmallVector< unsigned, 16 > RemainingCounts
static ScheduleDAGInstrs * createConveringSched(MachineSchedContext *C)
void placeDebugValues()
Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues.
static cl::opt< unsigned > ViewMISchedCutoff("view-misched-cutoff", cl::Hidden, cl::desc("Hide nodes with more predecessor/successor than cutoff"))
In some situations a few uninteresting nodes depend on nearly all other nodes in the graph...
void print(raw_ostream &OS, const SlotIndexes *=nullptr) const
print - Print out the MachineFunction in a format suitable for debugging to the specified stream...
unsigned getLatencyStallCycles(SUnit *SU)
Get the difference between the given SUnit's ready time and the current cycle.
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
SUnit * pickNode(bool &IsTopNode) override
Pick the best node to balance the schedule. Implements MachineSchedStrategy.
void pickNodeFromQueue(SchedBoundary &Zone, const CandPolicy &ZonePolicy, const RegPressureTracker &RPTracker, SchedCandidate &Candidate)
Pick the best candidate from the queue.
const TargetInstrInfo * TII
static bool isNodeHidden(const SUnit *Node)
const_iterator begin() const
void init(const MachineFunction *mf, const RegisterClassInfo *rci, const LiveIntervals *lis, const MachineBasicBlock *mbb, MachineBasicBlock::const_iterator pos, bool TrackLaneMasks, bool TrackUntiedDefs)
Setup the RegPressureTracker.
static cl::opt< unsigned > ReadyListLimit("misched-limit", cl::Hidden, cl::desc("Limit ready list to N instructions"), cl::init(256))
Avoid quadratic complexity in unusually large basic blocks by limiting the size of the ready lists...
unsigned getPSetOrMax() const
const std::vector< unsigned > & getRegSetPressureAtPos() const
Get the register set pressure at the current position, which may be less than the pressure across the...
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def...
void registerRoots() override
Notify this strategy that all roots have been released (including those that depend on EntrySU or Exi...
unsigned getReg() const
getReg - Returns the register number.
MachineBasicBlock::iterator CurrentBottom
The bottom of the unscheduled zone.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool addPred(const SDep &D, bool Required=true)
addPred - This adds the specified edge as a pred of the current node if not already.
bool operator<(int64_t V1, const APSInt &V2)
A raw_ostream that writes to an std::string.
void initQueues(ArrayRef< SUnit * > TopRoots, ArrayRef< SUnit * > BotRoots)
Release ExitSU predecessors and setup scheduler queues.
virtual const TargetInstrInfo * getInstrInfo() const
unsigned getMicroOpFactor() const
Multiply number of micro-ops by this factor to normalize it relative to other resources.
void postprocessDAG()
Apply each ScheduleDAGMutation step in order.
static LLVM_ATTRIBUTE_UNUSED Printable PrintLaneMask(LaneBitmask LaneMask)
Create Printable object to print LaneBitmasks on a raw_ostream.
SmallVector< SDep, 4 > Succs
unsigned getIssueWidth() const
Maximum number of micro-ops that may be scheduled per cycle.
MachineInstr * getInstructionFromIndex(SlotIndex index) const
Returns the instruction associated with the given index.
bool isBottomReady() const
static MachineSchedRegistry ILPMinRegistry("ilpmin","Schedule bottom-up for min ILP", createILPMinScheduler)
Arbitrary strong DAG edge (no real dependence).
void scheduleTree(unsigned SubtreeID)
Scheduler callback to update SubtreeConnectLevels when a tree is initially scheduled.
const SUnit * getNextClusterPred() const
This class implements an extremely fast bulk output stream that can only output to a stream...
StringRef getName() const
DefaultDOTGraphTraits - This class provides the default implementations of all of the DOTGraphTraits ...
void AddPred(SUnit *Y, SUnit *X)
AddPred - Updates the topological ordering to accommodate an edge to be added from SUnit X to SUnit Y...
MachineInstr * FirstDbgValue
MachineBasicBlock * BB
State specific to the current scheduling region.
unsigned getCurrMOps() const
Micro-ops issued in the current cycle.
bool canAddEdge(SUnit *SuccSU, SUnit *PredSU)
True if an edge can be added from PredSU to SuccSU without creating a cycle.
const TargetSchedModel * SchedModel
INITIALIZE_PASS(PostMachineScheduler,"postmisched","PostRA Machine Instruction Scheduler", false, false) PostMachineScheduler
bool readsReg() const
readsReg - Returns true if this operand reads the previous value of its register. ...
void setPos(MachineBasicBlock::const_iterator Pos)
bool isValid() const
Check for null.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
VNInfo * valueIn() const
Return the value that is live-in to the instruction.
RegisterPressure & getPressure()
Get the resulting register pressure over the traversed region.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
MachineRegisterInfo & MRI
std::vector< SUnit > SUnits
PriorityQueue - This class behaves like std::priority_queue and provides a few additional convenience...
ProcResIter getWriteProcResBegin(const MCSchedClassDesc *SC) const
SlotIndex - An opaque wrapper around machine indexes.
Printable PrintVRegOrUnit(unsigned VRegOrUnit, const TargetRegisterInfo *TRI)
Create Printable object to print virtual registers and physical registers on a raw_ostream.
const char * getResourceName(unsigned PIdx) const
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
static bool tryLatency(GenericSchedulerBase::SchedCandidate &TryCand, GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary &Zone)
RegPressureTracker TopRPTracker
RegPressureTracker RPTracker
void setBest(SchedCandidate &Best)
virtual HazardType getHazardType(SUnit *m, int Stalls=0)
getHazardType - Return the hazard type of emitting this node.
void pickNodeFromQueue(SchedCandidate &Cand)
unsigned getResourceCount(unsigned ResIdx) const
void dump(const ScheduleDAG *G) const
SUnit - Scheduling unit.
void getUpwardPressureDelta(const MachineInstr *MI, PressureDiff &PDiff, RegPressureDelta &Delta, ArrayRef< PressureChange > CriticalPSets, ArrayRef< unsigned > MaxPressureLimit) const
This is the fast version of querying register pressure that does not directly depend on current liven...
VNInfo * getVNInfoBefore(SlotIndex Idx) const
getVNInfoBefore - Return the VNInfo that is live up to but not necessarilly including Idx...
void handleMove(MachineInstr &MI, bool UpdateFlags=false)
handleMove - call this method to notify LiveIntervals that instruction 'mi' has been moved within a b...
SUnit - Scheduling unit. This is a node in the scheduling DAG.
void addMutation(std::unique_ptr< ScheduleDAGMutation > Mutation)
Add a postprocessing step to the DAG builder.
cl::opt< bool > ForceTopDown
bool isResourceLimited() const
static cl::opt< bool > EnableMemOpCluster("misched-cluster", cl::Hidden, cl::desc("Enable memop clustering."), cl::init(true))