75 #ifndef LLVM_CODEGEN_MACHINESCHEDULER_H
76 #define LLVM_CODEGEN_MACHINESCHEDULER_H
106 class MachineDominatorTree;
107 class MachineLoopInfo;
108 class RegisterClassInfo;
109 class SchedDFSResult;
110 class ScheduleHazardRecognizer;
192 virtual void anchor();
200 unsigned NumRegionInstrs) {}
255 std::vector<std::unique_ptr<ScheduleDAGMutation>>
Mutations;
299 Mutations.push_back(std::move(Mutation));
322 unsigned regioninstrs)
override;
416 std::unique_ptr<MachineSchedStrategy> S)
465 unsigned regioninstrs)
override;
497 const std::vector<unsigned> &NewMaxPressure);
518 std::vector<SUnit*> Queue;
523 unsigned getID()
const {
return ID; }
530 bool empty()
const {
return Queue.empty(); }
534 unsigned size()
const {
return Queue.size(); }
552 (*I)->NodeQueueId &= ~ID;
554 unsigned idx =
I - Queue.begin();
556 return Queue.begin() + idx;
624 unsigned MinReadyCycle;
627 unsigned ExpectedLatency;
632 unsigned DependentLatency;
636 unsigned RetiredMOps;
646 unsigned MaxExecutedResCount;
649 unsigned ZoneCritResIdx;
652 bool IsResourceLimited;
661 unsigned MaxObservedStall;
698 return std::max(ExpectedLatency, CurrCycle);
706 return ExecutedResCounts[ResIdx];
722 MaxExecutedResCount);
748 unsigned countResource(
unsigned PIdx,
unsigned Cycles,
unsigned ReadyCycle);
796 return !(*
this == RHS);
895 unsigned NumRegionInstrs)
override;
952 SchedCandidate &TryCand,
960 SchedCandidate &Candidate);
983 unsigned NumRegionInstrs)
override {
1010 BotRoots.push_back(SU);
1014 void tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand);
1027 std::unique_ptr<ScheduleDAGMutation>
1029 const TargetRegisterInfo *TRI);
1031 std::unique_ptr<ScheduleDAGMutation>
1033 const TargetRegisterInfo *TRI);
1035 std::unique_ptr<ScheduleDAGMutation>
1038 std::unique_ptr<ScheduleDAGMutation>
1040 const TargetRegisterInfo *TRI);
1044 #endif // LLVM_CODEGEN_MACHINESCHEDULER_H
VReg2SUnitMultiMap VRegUses
Maps vregs to the SUnits of their uses in the current scheduling region.
void computeDFSResult()
Compute a DFSResult after DAG building is complete, and before any queue comparisons.
void releaseSucc(SUnit *SU, SDep *SuccEdge)
ReleaseSucc - Decrement the NumPredsLeft count of a successor.
void initPolicy(MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, unsigned NumRegionInstrs) override
Optionally override the per-region scheduling policy.
void schedNode(SUnit *SU, bool IsTopNode) override
Called after ScheduleDAGMI has scheduled an instruction and updated scheduled/remaining flags in the ...
virtual void initialize(ScheduleDAGMI *DAG)=0
Initialize the strategy after building the DAG for a new region.
Base class for GenericScheduler.
virtual ~MachineSchedContext()
Each Scheduling boundary is associated with ready queues.
PostGenericScheduler - Interface to the scheduling algorithm used by ScheduleDAGMI.
GenericSchedulerBase(const MachineSchedContext *C)
const MachineSchedContext * Context
SUnit * pickNodeBidirectional(bool &IsTopNode)
Pick the best candidate node from either the top or bottom queue.
virtual void releaseTopNode(SUnit *SU)=0
When all predecessor dependencies have been resolved, free this node for top-down scheduling...
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
void init(ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel)
bool isInQueue(SUnit *SU) const
unsigned getCurrCycle() const
Number of cycles to issue the instructions scheduled in this zone.
ScheduleDAGTopologicalSort Topo
Topo - A topological ordering for SUnits which permits fast IsReachable and similar queries...
ScheduleDAGMI(MachineSchedContext *C, std::unique_ptr< MachineSchedStrategy > S, bool RemoveKillFlags)
MachineBasicBlock::iterator CurrentTop
The top of the unscheduled zone.
const MachineLoopInfo * MLI
bool isTrackingPressure() const
Return true if register pressure tracking is enabled.
void releaseTopNode(SUnit *SU) override
When all predecessor dependencies have been resolved, free this node for top-down scheduling...
unsigned computeCyclicCriticalPath()
Compute the cyclic critical path through the DAG.
A global registry used in conjunction with static constructors to make pluggable components (like tar...
void traceCandidate(const SchedCandidate &Cand)
unsigned DemandedResources
bool ShouldTrackPressure
Register pressure in this region computed by initRegPressure.
void *(* MachinePassCtor)()
MachineBasicBlock::iterator begin() const
begin - Return an iterator to the top of the current scheduling region.
ScheduleDAGMI * createGenericSchedPostRA(MachineSchedContext *C)
Create a generic scheduler with no vreg liveness or DAG mutation passes.
MachineSchedRegistry(const char *N, const char *D, ScheduleDAGCtor C)
Summarize the unscheduled region.
void reset(const CandPolicy &NewPolicy)
static void setListener(MachinePassRegistryListener *L)
virtual bool shouldTrackPressure() const
Check if pressure tracking is needed before building the DAG and initializing this strategy...
MachineSchedRegistry provides a selection of available machine instruction schedulers.
RegisterClassInfo * RegClassInfo
MachineBasicBlock::iterator top() const
const IntervalPressure & getTopPressure() const
Get current register pressure for the top scheduled instructions.
const RegPressureTracker & getTopRPTracker() const
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
std::unique_ptr< MachineSchedStrategy > SchedImpl
const RegPressureTracker & getBotRPTracker() const
MachineSchedRegistry * getNext() const
void updateQueues(SUnit *SU, bool IsTopNode)
Update scheduler DAG and queues after scheduling an instruction.
virtual void schedNode(SUnit *SU, bool IsTopNode)=0
Notify MachineSchedStrategy that ScheduleDAGMI has scheduled an instruction and updated scheduled/rem...
ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules machine instructions while...
unsigned getDependentLatency() const
unsigned NumInstrsScheduled
The number of instructions scheduled so far.
unsigned getHeight() const
getHeight - Return the height of this node, which is the length of the maximum path down to any node ...
ArrayRef< SUnit * > elements()
BitVector & getScheduledTrees()
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Provide an instruction scheduling machine model to CodeGen passes.
const HexagonInstrInfo * TII
const TargetPassConfig * PassConfig
const IntervalPressure & getBotPressure() const
Get current register pressure for the bottom scheduled instructions.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
void scheduleTree(unsigned SubtreeID) override
Scheduler callback to notify that a new subtree is scheduled.
PressureDiff & getPressureDiff(const SUnit *SU)
std::vector< std::unique_ptr< ScheduleDAGMutation > > Mutations
Ordered list of DAG postprocessing steps.
Target-Independent Code Generator Pass Configuration Options.
bool IsAcyclicLatencyLimited
ScheduleDAGCtor FunctionPassCtor
void initPolicy(MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, unsigned NumRegionInstrs) override
Initialize the per-region scheduling policy.
bool shouldTrackLaneMasks() const override
Returns true if lanemasks should be tracked.
bool ShouldTrackLaneMasks
static MachineSchedRegistry * getList()
Compute the values of each DAG node for various metrics during DFS.
void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs) override
Implement the ScheduleDAGInstrs interface for handling the next scheduling region.
MachineBasicBlock::iterator LiveRegionEnd
unsigned getCriticalCount() const
Get the scaled count of scheduled micro-ops and resources, including executed resources.
void bumpCycle(unsigned NextCycle)
Move the boundary of scheduled code by one cycle.
unsigned countResource(unsigned PIdx, unsigned Cycles, unsigned ReadyCycle)
Add the given processor resource to this scheduled zone.
void dumpPolicy() override
void scheduleMI(SUnit *SU, bool IsTopNode)
Move an instruction and update register pressure.
RegPressureTracker BotRPTracker
bool operator!=(const SchedResourceDelta &RHS) const
const SchedDFSResult * getDFSResult() const
Return a non-null DFS result if the scheduling strategy initialized it.
void buildDAGWithRegPressure()
Call ScheduleDAGInstrs::buildSchedGraph with register pressure tracking enabled.
bool DisconnectedComponentsRenamed
True if disconnected subregister components are already renamed.
Function Alias Analysis false
MachineSchedPolicy RegionPolicy
std::vector< PressureChange > RegionCriticalPSets
List of pressure sets that exceed the target's pressure limit before scheduling, listed in increasing...
static const char * getReasonStr(GenericSchedulerBase::CandReason Reason)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
void releasePending()
Release pending ready nodes in to the available queue.
virtual void initPolicy(MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, unsigned NumRegionInstrs)
Optionally override the per-region scheduling policy.
ScheduleDAGMILive * createGenericSchedLive(MachineSchedContext *C)
Create the standard converging machine scheduler.
MachinePassRegistry - Track the registration of machine passes.
std::unique_ptr< ScheduleDAGMutation > createMacroFusionDAGMutation(const TargetInstrInfo *TII)
void collectVRegUses(SUnit &SU)
void bumpNode(SUnit *SU)
Move the boundary of scheduled code by one SUnit.
SUnit * pickNode(bool &IsTopNode) override
Pick the next node to schedule.
void initCandidate(SchedCandidate &Cand, SUnit *SU, bool AtTop, const RegPressureTracker &RPTracker, RegPressureTracker &TempTracker)
void checkAcyclicLatency()
Set IsAcyclicLatencyLimited if the acyclic path is longer than the cyclic critical path by more cycle...
void incExecutedResources(unsigned PIdx, unsigned Count)
virtual void registerRoots()
Notify this strategy that all roots have been released (including those that depend on EntrySU or Exi...
SchedCandidate(const CandPolicy &Policy)
std::vector< SUnit * >::iterator iterator
SDep - Scheduling dependency.
RegisterClassInfo * RegClassInfo
ScheduleHazardRecognizer * HazardRec
unsigned getNextResourceCycle(unsigned PIdx, unsigned Cycles)
Compute the next cycle at which the given processor resource can be scheduled.
void initQueues(ArrayRef< SUnit * > TopRoots, ArrayRef< SUnit * > BotRoots)
Release ExitSU predecessors and setup scheduler queues.
CandReason
Represent the type of SchedCandidate found within a single queue.
Helpers for implementing custom MachineSchedStrategy classes.
RegisterPressure computed within a region of instructions delimited by TopIdx and BottomIdx...
bool operator==(const SchedResourceDelta &RHS) const
HazardRecognizer - This determines whether or not an instruction can be issued this cycle...
unsigned getScheduledLatency() const
Get the number of latency cycles "covered" by the scheduled instructions.
void dumpSchedule() const
dump the scheduled Sequence.
bool RemoveKillFlags
True if the DAG builder should remove kill flags (in preparation for rescheduling).
bool operator==(const CandPolicy &RHS) const
void removeReady(SUnit *SU)
Remove SU from the ready set for this boundary.
unsigned getZoneCritResIdx() const
void updatePressureDiffs(ArrayRef< RegisterMaskPair > LiveUses)
Update the PressureDiff array for liveness after scheduling this instruction.
unsigned getLatencyFactor() const
Multiply cycle count by this factor to normalize it relative to other resources.
MachinePassRegistryNode * getNext() const
bool addEdge(SUnit *SuccSU, const SDep &PredDep)
Add a DAG edge to the given SU with the given predecessor dependence data.
bool ShouldTrackLaneMasks
Track LaneMasks to allow reordering of independent subregister writes of the same vreg...
static MachinePassRegistry Registry
Store the state used by GenericScheduler heuristics, required for the lifetime of one invocation of p...
bool checkHazard(SUnit *SU)
Does this SU have a hazard within the current instruction group.
void tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand, SchedBoundary *Zone)
Apply a set of heursitics to a new candidate.
void releasePred(SUnit *SU, SDep *PredEdge)
ReleasePred - Decrement the NumSuccsLeft count of a predecessor.
Track the current register pressure at some position in the instruction stream, and remember the high...
static const unsigned End
virtual void releaseBottomNode(SUnit *SU)=0
When all successor dependencies have been resolved, free this node for bottom-up scheduling.
void findRootsAndBiasEdges(SmallVectorImpl< SUnit * > &TopRoots, SmallVectorImpl< SUnit * > &BotRoots)
Policy for scheduling the next instruction in the candidate's zone.
const TargetSchedModel * SchedModel
List of PressureChanges in order of increasing, unique PSetID.
SchedResourceDelta ResDelta
const TargetRegisterInfo * TRI
std::unique_ptr< ScheduleDAGMutation > createStoreClusterDAGMutation(const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
SchedDFSResult * DFSResult
Information about DAG subtrees.
PressureDiffs SUPressureDiffs
const IntervalPressure & getRegPressure() const
Get register pressure for the entire scheduling region before scheduling.
void dumpScheduledState()
virtual void dumpPolicy()
LiveIntervals * getLIS() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
void releaseSuccessors(SUnit *SU)
releaseSuccessors - Call releaseSucc on each of SU's successors.
const SUnit * NextClusterSucc
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void viewGraph() override
Out-of-line implementation with no arguments is handy for gdb.
void setPolicy(CandPolicy &Policy, bool IsPostRA, SchedBoundary &CurrZone, SchedBoundary *OtherZone)
Set the CandPolicy given a scheduling zone given the current resources and latencies inside and outsi...
SchedBoundary(unsigned ID, const Twine &Name)
Pending queues extend the ready queues with the same ID and the PendingFlag set.
void init(ScheduleDAGMI *dag, const TargetSchedModel *smodel, SchedRemainder *rem)
virtual bool shouldTrackLaneMasks() const
Returns true if lanemasks should be tracked.
std::unique_ptr< ScheduleDAGMutation > createLoadClusterDAGMutation(const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
const SUnit * NextClusterPred
Record the next node in a scheduled cluster.
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
void registerRoots() override
Notify this strategy that all roots have been released (including those that depend on EntrySU or Exi...
void reschedulePhysRegCopies(SUnit *SU, bool isTop)
GenericScheduler shrinks the unscheduled zone using heuristics to balance the schedule.
unsigned getExecutedCount() const
Get a scaled count for the minimum execution time of the scheduled micro-ops that are ready to execut...
ScheduleDAGInstrs *(* ScheduleDAGCtor)(MachineSchedContext *)
bool shouldTrackPressure() const override
Check if pressure tracking is needed before building the DAG and initializing this strategy...
const MachineLoopInfo * MLI
void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs) override
Implement the ScheduleDAGInstrs interface for handling the next scheduling region.
void releaseNode(SUnit *SU, unsigned ReadyCycle)
bool hasVRegLiveness() const override
Return true if this DAG supports VReg liveness and RegPressure.
void updateScheduledPressure(const SUnit *SU, const std::vector< unsigned > &NewMaxPressure)
std::unique_ptr< ScheduleDAGMutation > createCopyConstrainDAGMutation(const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
void initResourceDelta(const ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
~ScheduleDAGMILive() override
unsigned getOtherResourceCount(unsigned &OtherCritIdx)
PostGenericScheduler(const MachineSchedContext *C)
void tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand)
Apply a set of heursitics to a new candidate for PostRA scheduling.
ReadyQueue(unsigned id, const Twine &name)
MachinePassRegistryListener - Listener to adds and removals of nodes in registration list...
MachineBasicBlock::iterator bottom() const
void moveInstruction(MachineInstr *MI, MachineBasicBlock::iterator InsertPos)
Change the position of an instruction within the basic block and update live ranges and region bounda...
ScheduleDAGMILive(MachineSchedContext *C, std::unique_ptr< MachineSchedStrategy > S)
MachineBasicBlock::iterator end() const
end - Return an iterator to the bottom of the current scheduling region.
virtual bool hasVRegLiveness() const
Return true if this DAG supports VReg liveness and RegPressure.
unsigned getDepth() const
getDepth - Return the depth of this node, which is the length of the maximum path up to any node whic...
void schedNode(SUnit *SU, bool IsTopNode) override
Update the scheduler's state after scheduling a node.
const SUnit * getNextClusterSucc() const
virtual void scheduleTree(unsigned SubtreeID)
Scheduler callback to notify that a new subtree is scheduled.
void releasePredecessors(SUnit *SU)
releasePredecessors - Call releasePred on each of SU's predecessors.
unsigned getUnscheduledLatency(SUnit *SU) const
void initialize(ScheduleDAGMI *dag) override
Initialize the strategy after building the DAG for a new region.
virtual SUnit * pickNode(bool &IsTopNode)=0
Pick the next node to schedule, or return NULL.
ScheduleDAGInstrs - A ScheduleDAG subclass for scheduling lists of MachineInstrs. ...
Define a generic scheduling policy for targets that don't provide their own MachineSchedStrategy.
Representation of each machine instruction.
const MachineDominatorTree * MDT
void initialize(ScheduleDAGMI *Dag) override
Initialize the strategy after building the DAG for a new region.
bool operator!=(const CandPolicy &RHS) const
SUnit * pickOnlyChoice()
Call this before applying any other heuristics to the Available queue.
Status of an instruction's critical resource consumption.
~ScheduleDAGMI() override
GenericScheduler(const MachineSchedContext *C)
bool shouldTrackPressure() const override
PostRA scheduling does not track pressure.
SchedCandidate BotCand
Candidate last picked from Bot boundary.
cl::opt< bool > ForceBottomUp
unsigned findMaxLatency(ArrayRef< SUnit * > ReadySUs)
MachineSchedContext provides enough context from the MachineScheduler pass for the target to instanti...
~PostGenericScheduler() override=default
MachineSchedStrategy - Interface to the scheduling algorithm used by ScheduleDAGMI.
MachinePassRegistryNode - Machine pass node stored in registration list.
virtual ~MachineSchedStrategy()=default
SmallVector< unsigned, 16 > RemainingCounts
void placeDebugValues()
Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues.
unsigned getLatencyStallCycles(SUnit *SU)
Get the difference between the given SUnit's ready time and the current cycle.
void schedule() override
Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.
SUnit * pickNode(bool &IsTopNode) override
Pick the best node to balance the schedule. Implements MachineSchedStrategy.
void pickNodeFromQueue(SchedBoundary &Zone, const CandPolicy &ZonePolicy, const RegPressureTracker &RPTracker, SchedCandidate &Candidate)
Pick the best candidate from the queue.
const std::vector< PressureChange > & getRegionCriticalPSets() const
IntervalPressure TopPressure
The top of the unscheduled zone.
void releaseTopNode(SUnit *SU) override
When all predecessor dependencies have been resolved, free this node for top-down scheduling...
void registerRoots() override
Notify this strategy that all roots have been released (including those that depend on EntrySU or Exi...
MachineBasicBlock::iterator CurrentBottom
The bottom of the unscheduled zone.
Store the effects of a change in pressure on things that MI scheduler cares about.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void initQueues(ArrayRef< SUnit * > TopRoots, ArrayRef< SUnit * > BotRoots)
Release ExitSU predecessors and setup scheduler queues.
unsigned getMicroOpFactor() const
Multiply number of micro-ops by this factor to normalize it relative to other resources.
void postprocessDAG()
Apply each ScheduleDAGMutation step in order.
SchedCandidate TopCand
Candidate last picked from Top boundary.
const SUnit * getNextClusterPred() const
StringRef getName() const
IntervalPressure RegPressure
unsigned getCurrMOps() const
Micro-ops issued in the current cycle.
bool canAddEdge(SUnit *SuccSU, SUnit *PredSU)
True if an edge can be added from PredSU to SuccSU without creating a cycle.
StringRef - Represent a constant reference to a string, i.e.
const TargetSchedModel * SchedModel
bool DisableLatencyHeuristic
IntervalPressure BotPressure
The bottom of the unscheduled zone.
std::vector< SUnit > SUnits
ScheduleDAGTopologicalSort is a class that computes a topological ordering for SUnits and provides me...
void releaseBottomNode(SUnit *SU) override
When all successor dependencies have been resolved, free this node for bottom-up scheduling.
RegPressureTracker TopRPTracker
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
RegPressureTracker RPTracker
void setBest(SchedCandidate &Best)
void pickNodeFromQueue(SchedCandidate &Cand)
unsigned getResourceCount(unsigned ResIdx) const
SUnit - Scheduling unit. This is a node in the scheduling DAG.
void addMutation(std::unique_ptr< ScheduleDAGMutation > Mutation)
Add a postprocessing step to the DAG builder.
cl::opt< bool > ForceTopDown
void releaseBottomNode(SUnit *SU) override
When all successor dependencies have been resolved, free this node for bottom-up scheduling.
bool isResourceLimited() const