48 #define DEBUG_TYPE "post-RA-sched"
50 STATISTIC(NumNoops,
"Number of noops inserted");
51 STATISTIC(NumStalls,
"Number of pipeline stalls");
52 STATISTIC(NumFixedAnti,
"Number of fixed anti-dependencies");
59 cl::desc(
"Enable scheduling after register allocation"),
63 cl::desc(
"Break post-RA scheduling anti-dependencies: "
64 "\"critical\", \"all\", or \"none\""),
70 cl::desc(
"Debug control MBBs that are scheduled"),
74 cl::desc(
"Debug control MBBs that are scheduled"),
107 bool enablePostRAScheduler(
123 std::vector<SUnit*> PendingQueue;
138 std::vector<std::unique_ptr<ScheduleDAGMutation>> Mutations;
147 SchedulePostRATDList(
153 ~SchedulePostRATDList()
override;
161 void setEndIndex(
unsigned EndIdx) { EndIndex = EndIdx; }
167 unsigned regioninstrs)
override;
170 void exitRegion()
override;
174 void schedule()
override;
185 void finishBlock()
override;
189 void postprocessDAG();
191 void ReleaseSucc(
SUnit *SU,
SDep *SuccEdge);
192 void ReleaseSuccessors(
SUnit *SU);
193 void ScheduleNodeTopDown(
SUnit *SU,
unsigned CurCycle);
194 void ListScheduleTopDown();
196 void dumpSchedule()
const;
197 void emitNoop(
unsigned CurCycle);
204 "Post RA top-down list latency scheduler",
false,
false)
206 SchedulePostRATDList::SchedulePostRATDList(
214 MF.getSubtarget().getInstrItineraryData();
216 MF.getSubtarget().getInstrInfo()->CreateTargetPostRAHazardRecognizer(
218 MF.getSubtarget().getPostRAMutations(Mutations);
221 MRI.tracksLiveness()) &&
222 "Live-ins must be accurate for anti-dependency breaking");
230 SchedulePostRATDList::~SchedulePostRATDList() {
239 unsigned regioninstrs) {
245 void SchedulePostRATDList::exitRegion() {
247 dbgs() <<
"*** Final schedule ***\n";
254 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
256 void SchedulePostRATDList::dumpSchedule()
const {
257 for (
unsigned i = 0, e =
Sequence.size();
i != e;
i++) {
261 dbgs() <<
"**** NOOP ****\n";
266 bool PostRAScheduler::enablePostRAScheduler(
288 AliasAnalysis *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
291 RegClassInfo.runOnMachineFunction(Fn);
300 AntiDepMode, CriticalPathRCs))
314 SchedulePostRATDList
Scheduler(Fn, MLI, AA, RegClassInfo, AntiDepMode,
318 for (
auto &
MBB : Fn) {
322 static int bbcnt = 0;
325 dbgs() <<
"*** DEBUG scheduling " << Fn.getName()
336 unsigned Count =
MBB.
size(), CurrentCount = Count;
344 Scheduler.enterRegion(&
MBB,
I, Current, CurrentCount - Count);
350 CurrentCount = Count;
357 assert(Count == 0 &&
"Instruction count mismatch!");
359 "Instruction count mismatch!");
386 AntiDepBreak->StartBlock(BB);
391 void SchedulePostRATDList::schedule() {
397 AntiDepBreak->BreakAntiDependencies(SUnits, RegionBegin, RegionEnd,
398 EndIndex, DbgValues);
410 NumFixedAnti += Broken;
416 DEBUG(
dbgs() <<
"********** List Scheduling **********\n");
418 for (
const SUnit &SU : SUnits) {
424 AvailableQueue.initNodes(SUnits);
425 ListScheduleTopDown();
426 AvailableQueue.releaseState();
432 void SchedulePostRATDList::Observe(
MachineInstr &MI,
unsigned Count) {
434 AntiDepBreak->Observe(MI, Count, EndIndex);
439 void SchedulePostRATDList::finishBlock() {
441 AntiDepBreak->FinishBlock();
448 void SchedulePostRATDList::postprocessDAG() {
449 for (
auto &M : Mutations)
459 void SchedulePostRATDList::ReleaseSucc(
SUnit *SU,
SDep *SuccEdge) {
468 dbgs() <<
"*** Scheduling failed! ***\n";
470 dbgs() <<
" has been released too many times!\n";
490 PendingQueue.push_back(SuccSU);
494 void SchedulePostRATDList::ReleaseSuccessors(
SUnit *SU) {
497 ReleaseSucc(SU, &*
I);
504 void SchedulePostRATDList::ScheduleNodeTopDown(
SUnit *SU,
unsigned CurCycle) {
505 DEBUG(
dbgs() <<
"*** Scheduling [" << CurCycle <<
"]: ");
510 "Node scheduled above its depth!");
513 ReleaseSuccessors(SU);
515 AvailableQueue.scheduledNode(SU);
519 void SchedulePostRATDList::emitNoop(
unsigned CurCycle) {
520 DEBUG(
dbgs() <<
"*** Emitting noop in cycle " << CurCycle <<
'\n');
521 HazardRec->EmitNoop();
528 void SchedulePostRATDList::ListScheduleTopDown() {
529 unsigned CurCycle = 0;
538 ReleaseSuccessors(&EntrySU);
541 for (
unsigned i = 0, e = SUnits.size();
i != e; ++
i) {
544 AvailableQueue.push(&SUnits[
i]);
545 SUnits[
i].isAvailable =
true;
551 bool CycleHasInsts =
false;
555 std::vector<SUnit*> NotReady;
557 while (!AvailableQueue.empty() || !PendingQueue.empty()) {
560 unsigned MinDepth = ~0u;
561 for (
unsigned i = 0, e = PendingQueue.size();
i != e; ++
i) {
562 if (PendingQueue[
i]->getDepth() <= CurCycle) {
563 AvailableQueue.push(PendingQueue[
i]);
564 PendingQueue[
i]->isAvailable =
true;
565 PendingQueue[
i] = PendingQueue.back();
566 PendingQueue.pop_back();
568 }
else if (PendingQueue[i]->getDepth() < MinDepth)
569 MinDepth = PendingQueue[
i]->getDepth();
572 DEBUG(
dbgs() <<
"\n*** Examining Available\n"; AvailableQueue.dump(
this));
574 SUnit *FoundSUnit =
nullptr, *NotPreferredSUnit =
nullptr;
575 bool HasNoopHazards =
false;
576 while (!AvailableQueue.empty()) {
577 SUnit *CurSUnit = AvailableQueue.pop();
580 HazardRec->getHazardType(CurSUnit, 0);
582 if (HazardRec->ShouldPreferAnother(CurSUnit)) {
583 if (!NotPreferredSUnit) {
588 NotPreferredSUnit = CurSUnit;
592 FoundSUnit = CurSUnit;
600 NotReady.push_back(CurSUnit);
606 if (NotPreferredSUnit) {
608 DEBUG(
dbgs() <<
"*** Will schedule a non-preferred instruction...\n");
609 FoundSUnit = NotPreferredSUnit;
611 AvailableQueue.push(NotPreferredSUnit);
614 NotPreferredSUnit =
nullptr;
618 if (!NotReady.empty()) {
619 AvailableQueue.push_all(NotReady);
626 unsigned NumPreNoops = HazardRec->PreEmitNoops(FoundSUnit);
627 for (
unsigned i = 0; i != NumPreNoops; ++
i)
631 ScheduleNodeTopDown(FoundSUnit, CurCycle);
632 HazardRec->EmitInstruction(FoundSUnit);
633 CycleHasInsts =
true;
634 if (HazardRec->atIssueLimit()) {
635 DEBUG(
dbgs() <<
"*** Max instructions per cycle " << CurCycle <<
'\n');
636 HazardRec->AdvanceCycle();
638 CycleHasInsts =
false;
642 DEBUG(
dbgs() <<
"*** Finished cycle " << CurCycle <<
'\n');
643 HazardRec->AdvanceCycle();
644 }
else if (!HasNoopHazards) {
647 DEBUG(
dbgs() <<
"*** Stall in cycle " << CurCycle <<
'\n');
648 HazardRec->AdvanceCycle();
658 CycleHasInsts =
false;
663 unsigned ScheduledNodes = VerifyScheduledDAG(
false);
665 for (
unsigned i = 0, e =
Sequence.size(); i != e; ++
i)
669 "The number of nodes scheduled doesn't match the expected number!");
674 void SchedulePostRATDList::EmitSchedule() {
675 RegionBegin = RegionEnd;
679 BB->
splice(RegionEnd, BB, FirstDbgValue);
682 for (
unsigned i = 0, e =
Sequence.size(); i != e; i++) {
692 RegionBegin = std::prev(RegionEnd);
696 for (std::vector<std::pair<MachineInstr *, MachineInstr *> >::iterator
697 DI = DbgValues.end(), DE = DbgValues.begin();
DI != DE; --
DI) {
698 std::pair<MachineInstr *, MachineInstr *>
P = *std::prev(DI);
701 BB->
splice(++OrigPrivMI, BB, DbgValue);
704 FirstDbgValue =
nullptr;
const_iterator end(StringRef path)
Get end iterator over path.
virtual void finishBlock()
finishBlock - Clean up after scheduling in the given block.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
STATISTIC(NumFunctions,"Total number of functions")
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
MachineInstr * getInstr() const
getInstr - Return the representative MachineInstr for this SUnit.
const_iterator begin(StringRef path)
Get begin iterator over path.
static cl::opt< int > DebugDiv("postra-sched-debugdiv", cl::desc("Debug control MBBs that are scheduled"), cl::init(0), cl::Hidden)
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
virtual bool enablePostRAScheduler() const
True if the subtarget should run a scheduler after register allocation.
virtual CodeGenOpt::Level getOptLevelToEnablePostRAScheduler() const
virtual void startBlock(MachineBasicBlock *BB)
startBlock - Prepare to perform scheduling in the given block.
virtual AntiDepBreakMode getAntiDepBreakMode() const
void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
Insert a noop into the instruction stream at the specified point.
CodeGenOpt::Level getOptLevel() const
AnalysisUsage & addRequired()
unsigned getBundleSize() const
Return the number of instructions inside the MI bundle, excluding the bundle header.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
bool isWeak() const
isWeak - Test if this a weak dependence.
const HexagonInstrInfo * TII
This class works in conjunction with the post-RA scheduler to rename registers to break register anti...
static cl::opt< bool > EnablePostRAScheduler("post-RA-scheduler", cl::desc("Enable scheduling after register allocation"), cl::init(false), cl::Hidden)
INITIALIZE_PASS(PostRAScheduler,"post-RA-sched","Post RA top-down list latency scheduler", false, false) SchedulePostRATDList
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Target-Independent Code Generator Pass Configuration Options.
Itinerary data supplied by a subtarget to be used by a target.
virtual void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs)
Initialize the scheduler state for the next scheduling region.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
TargetInstrInfo - Interface to description of machine instruction set.
void setDepthToAtLeast(unsigned NewDepth)
setDepthToAtLeast - If NewDepth is greater than this node's depth value, set it to be the new depth v...
SDep - Scheduling dependency.
initializer< Ty > init(const Ty &Val)
unsigned const MachineRegisterInfo * MRI
void clearDAG()
clearDAG - clear the DAG state (between regions).
HazardRecognizer - This determines whether or not an instruction can be issued this cycle...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Represent the analysis usage information of a pass.
virtual void exitRegion()
Notify that the scheduler has finished scheduling the current region.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
char & PostRASchedulerID
createPostRAScheduler - This pass performs post register allocation scheduling.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
void setPreservesCFG()
This function should be called by the pass, iff they do not:
static cl::opt< std::string > EnableAntiDepBreaking("break-anti-dependencies", cl::desc("Break post-RA scheduling anti-dependencies: ""\"critical\", \"all\", or \"none\""), cl::init("none"), cl::Hidden)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
Test if the given instruction should be considered a scheduling boundary.
machine Machine Instruction Scheduler
unsigned getDepth() const
getDepth - Return the depth of this node, which is the length of the maximum path up to any node whic...
MachineFunctionProperties & set(Property P)
TargetSubtargetInfo - Generic base class for all target subtargets.
static cl::opt< int > DebugMod("postra-sched-debugmod", cl::desc("Debug control MBBs that are scheduled"), cl::init(0), cl::Hidden)
ScheduleDAGInstrs - A ScheduleDAG subclass for scheduling lists of MachineInstrs. ...
Representation of each machine instruction.
virtual ~AntiDepBreaker()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
bool isCall(QueryType Type=AnyInBundle) const
Sequence
A sequence of states that a pointer may go through in which an objc_retain and objc_release are actua...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
virtual void getCriticalPathRCs(RegClassVector &CriticalPathRCs) const
virtual const TargetInstrInfo * getInstrInfo() const
SmallVector< SDep, 4 > Succs
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
void dump(const ScheduleDAG *G) const
SUnit - Scheduling unit.
Properties which a MachineFunction may have at a given point in time.
SUnit - Scheduling unit. This is a node in the scheduling DAG.
This file describes how to lower LLVM code to machine code.