30 #define DEBUG_TYPE "pre-RA-sched"
32 STATISTIC(NumUnfolds,
"Number of nodes unfolded");
33 STATISTIC(NumDups,
"Number of duplicated nodes");
34 STATISTIC(NumPRCopies,
"Number of physical copies");
48 struct FastPriorityQueue {
51 bool empty()
const {
return Queue.empty(); }
58 if (empty())
return nullptr;
71 FastPriorityQueue AvailableQueue;
77 std::vector<SUnit*> LiveRegDefs;
78 std::vector<unsigned> LiveRegCycles;
84 void Schedule()
override;
94 void RemovePred(
SUnit *SU,
const SDep &D) {
99 void ReleasePred(
SUnit *SU,
SDep *PredEdge);
100 void ReleasePredecessors(
SUnit *SU,
unsigned CurCycle);
101 void ScheduleNodeBottomUp(
SUnit*,
unsigned);
103 void InsertCopiesAndMoveSuccs(
SUnit*,
unsigned,
108 void ListScheduleBottomUp();
111 bool forceUnitLatencies()
const override {
return true; }
117 void ScheduleDAGFast::Schedule() {
118 DEBUG(
dbgs() <<
"********** List Scheduling **********\n");
121 LiveRegDefs.resize(TRI->getNumRegs(),
nullptr);
122 LiveRegCycles.resize(TRI->getNumRegs(), 0);
125 BuildSchedGraph(
nullptr);
127 DEBUG(
for (
unsigned su = 0, e = SUnits.size(); su != e; ++su)
128 SUnits[su].dumpAll(
this));
131 ListScheduleBottomUp();
140 void ScheduleDAGFast::ReleasePred(
SUnit *SU,
SDep *PredEdge) {
145 dbgs() <<
"*** Scheduling failed! ***\n";
147 dbgs() <<
" has been released too many times!\n";
157 AvailableQueue.push(PredSU);
161 void ScheduleDAGFast::ReleasePredecessors(
SUnit *SU,
unsigned CurCycle) {
165 ReleasePred(SU, &*
I);
166 if (
I->isAssignedRegDep()) {
171 if (!LiveRegDefs[
I->getReg()]) {
173 LiveRegDefs[
I->getReg()] =
I->getSUnit();
174 LiveRegCycles[
I->getReg()] = CurCycle;
183 void ScheduleDAGFast::ScheduleNodeBottomUp(
SUnit *SU,
unsigned CurCycle) {
184 DEBUG(
dbgs() <<
"*** Scheduling [" << CurCycle <<
"]: ");
187 assert(CurCycle >= SU->
getHeight() &&
"Node scheduled below its height!");
191 ReleasePredecessors(SU, CurCycle);
196 if (
I->isAssignedRegDep()) {
197 if (LiveRegCycles[
I->getReg()] ==
I->getSUnit()->getHeight()) {
198 assert(NumLiveRegs > 0 &&
"NumLiveRegs is already zero!");
199 assert(LiveRegDefs[
I->getReg()] == SU &&
200 "Physical register dependency violated?");
202 LiveRegDefs[
I->getReg()] =
nullptr;
203 LiveRegCycles[
I->getReg()] = 0;
213 SUnit *ScheduleDAGFast::CopyAndMoveSuccessors(
SUnit *SU) {
222 bool TryUnfold =
false;
231 MVT VT =
Op.getNode()->getSimpleValueType(
Op.getResNo());
238 if (!
TII->unfoldMemoryOperand(*DAG, N, NewNodes))
242 assert(NewNodes.
size() == 2 &&
"Expected a load folding node!");
245 SDNode *LoadNode = NewNodes[0];
246 unsigned NumVals = N->getNumValues();
248 for (
unsigned i = 0;
i != NumVals; ++
i)
250 DAG->ReplaceAllUsesOfValueWith(
SDValue(SU->
getNode(), OldNumVals-1),
253 SUnit *NewSU = newSUnit(N);
254 assert(N->getNodeId() == -1 &&
"Node already inserted!");
270 bool isNewLoad =
true;
276 LoadSU = newSUnit(LoadNode);
289 else if (
I->getSUnit()->getNode() &&
290 I->getSUnit()->getNode()->isOperandOf(LoadNode))
304 RemovePred(SU, ChainPred);
306 AddPred(LoadSU, ChainPred);
308 for (
unsigned i = 0, e = LoadPreds.
size();
i != e; ++
i) {
309 const SDep &Pred = LoadPreds[
i];
310 RemovePred(SU, Pred);
312 AddPred(LoadSU, Pred);
315 for (
unsigned i = 0, e = NodePreds.
size();
i != e; ++
i) {
316 const SDep &Pred = NodePreds[
i];
317 RemovePred(SU, Pred);
318 AddPred(NewSU, Pred);
320 for (
unsigned i = 0, e = NodeSuccs.
size();
i != e; ++
i) {
324 RemovePred(SuccDep, D);
328 for (
unsigned i = 0, e = ChainSuccs.
size();
i != e; ++
i) {
329 SDep D = ChainSuccs[
i];
332 RemovePred(SuccDep, D);
359 if (!
I->isArtificial())
367 if (
I->isArtificial())
369 SUnit *SuccSU =
I->getSUnit();
375 DelDeps.
push_back(std::make_pair(SuccSU, D));
378 for (
unsigned i = 0, e = DelDeps.
size();
i != e; ++
i)
379 RemovePred(DelDeps[
i].first, DelDeps[
i].second);
387 void ScheduleDAGFast::InsertCopiesAndMoveSuccs(
SUnit *SU,
unsigned Reg,
391 SUnit *CopyFromSU = newSUnit(static_cast<SDNode *>(
nullptr));
395 SUnit *CopyToSU = newSUnit(static_cast<SDNode *>(
nullptr));
404 if (
I->isArtificial())
406 SUnit *SuccSU =
I->getSUnit();
411 DelDeps.
push_back(std::make_pair(SuccSU, *
I));
414 for (
unsigned i = 0, e = DelDeps.
size();
i != e; ++
i) {
415 RemovePred(DelDeps[
i].first, DelDeps[
i].second);
418 FromDep.setLatency(SU->
Latency);
419 AddPred(CopyFromSU, FromDep);
421 ToDep.setLatency(CopyFromSU->
Latency);
422 AddPred(CopyToSU, ToDep);
455 std::vector<SUnit*> &LiveRegDefs,
461 if (LiveRegDefs[*AI] && LiveRegDefs[*AI] != SU) {
462 if (RegAdded.
insert(*AI).second) {
475 bool ScheduleDAGFast::DelayForLiveRegsBottomUp(
SUnit *SU,
477 if (NumLiveRegs == 0)
484 if (
I->isAssignedRegDep()) {
486 RegAdded, LRegs, TRI);
493 unsigned NumOps = Node->getNumOperands();
494 if (Node->getOperand(NumOps-1).getValueType() ==
MVT::Glue)
499 cast<ConstantSDNode>(Node->getOperand(
i))->getZExtValue();
507 for (; NumVals; --NumVals, ++
i) {
508 unsigned Reg = cast<RegisterSDNode>(Node->getOperand(
i))->
getReg();
517 if (!Node->isMachineOpcode())
526 return !LRegs.
empty();
532 void ScheduleDAGFast::ListScheduleBottomUp() {
533 unsigned CurCycle = 0;
536 ReleasePredecessors(&ExitSU, CurCycle);
539 if (!SUnits.empty()) {
540 SUnit *RootSU = &SUnits[DAG->getRoot().getNode()->getNodeId()];
541 assert(RootSU->
Succs.empty() &&
"Graph root shouldn't have successors!");
543 AvailableQueue.push(RootSU);
551 while (!AvailableQueue.empty()) {
552 bool Delayed =
false;
554 SUnit *CurSU = AvailableQueue.pop();
557 if (!DelayForLiveRegsBottomUp(CurSU, LRegs))
560 LRegsMap.
insert(std::make_pair(CurSU, LRegs));
564 CurSU = AvailableQueue.pop();
570 if (Delayed && !CurSU) {
575 SUnit *TrySU = NotReady[0];
577 assert(LRegs.
size() == 1 &&
"Can't handle this yet!");
578 unsigned Reg = LRegs[0];
582 TRI->getMinimalPhysRegClass(Reg, VT);
592 SUnit *NewDef =
nullptr;
594 NewDef = CopyAndMoveSuccessors(LRDef);
595 if (!DestRC && !NewDef)
597 "register dependency!");
602 InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
606 NewDef = Copies.
back();
610 <<
" to SU #" << TrySU->
NodeNum <<
"\n");
611 LiveRegDefs[
Reg] = NewDef;
623 for (
unsigned i = 0, e = NotReady.
size();
i != e; ++
i) {
624 NotReady[
i]->isPending =
false;
627 AvailableQueue.push(NotReady[
i]);
632 ScheduleNodeBottomUp(CurSU, CurCycle);
640 VerifyScheduledSequence(
true);
655 void Schedule()
override;
664 void ScheduleNode(
SDNode *N);
668 void ScheduleDAGLinearize::ScheduleNode(
SDNode *N) {
682 if (
unsigned NumLeft = NumOps) {
683 SDNode *GluedOpN =
nullptr;
702 if (DI != GluedMap.
end() && DI->second !=
N)
707 assert(Degree > 0 &&
"Predecessor over-released!");
723 void ScheduleDAGLinearize::Schedule() {
724 DEBUG(
dbgs() <<
"********** DAG Linearization **********\n");
727 unsigned DAGSize = 0;
728 for (
SDNode &Node : DAG->allnodes()) {
740 GluedMap.insert(std::make_pair(N, User));
749 for (
unsigned i = 0, e = Glues.
size();
i != e; ++
i) {
751 SDNode *GUser = GluedMap[Glue];
767 ScheduleNode(DAG->getRoot().getNode());
776 dbgs() <<
"\n*** Final schedule ***\n";
780 unsigned NumNodes =
Sequence.size();
781 for (
unsigned i = 0;
i != NumNodes; ++
i) {
784 Emitter.EmitNode(N,
false,
false, VRBaseMap);
789 InsertPos = Emitter.getInsertPos();
790 return Emitter.getBlock();
799 return new ScheduleDAGFast(*IS->
MF);
804 return new ScheduleDAGLinearize(*IS->
MF);
void push_back(const T &Elt)
STATISTIC(NumFunctions,"Total number of functions")
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool isCommutable() const
Return true if this may be a 2- or 3-address instruction (of the form "X = op Y, Z, ..."), which produces the same result if Y and Z are exchanged.
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Describe properties that are true of each instruction in the target description file.
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
static bool isClobberKind(unsigned Flag)
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
void removePred(const SDep &D)
removePred - This removes the specified edge as a pred of the current node if it exists.
void setNodeId(int Id)
Set unique node id.
const TargetRegisterClass * CopyDstRC
EntryToken - This is the marker used to indicate the start of a region.
SmallVector< SDep, 4 > Preds
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
unsigned getHeight() const
getHeight - Return the height of this node, which is the length of the maximum path down to any node ...
struct fuzzer::@269 Flags
const HexagonInstrInfo * TII
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
Regular data dependence (aka true-dependence).
Reg
All possible values of the reg field in the ModR/M byte.
static bool isRegDefEarlyClobberKind(unsigned Flag)
INLINEASM - Represents an inline asm block.
LLVM_NODISCARD bool empty() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
ScheduleDAGSDNodes * createDAGLinearizer(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDAGLinearizer - This creates a "no-scheduling" scheduler which linearize the DAG using topologi...
const MCPhysReg * getImplicitDefs() const
Return a list of registers that are potentially written by any instance of this machine instruction...
static bool isRegDefKind(unsigned Flag)
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
TargetInstrInfo - Interface to description of machine instruction set.
SDep - Scheduling dependency.
SDNode * getNode() const
get the SDNode which holds the desired result
void setHeightToAtLeast(unsigned NewHeight)
setDepthToAtLeast - If NewDepth is greater than this node's depth value, set it to be the new height ...
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
size_t use_size() const
Return the number of uses of this node.
MVT - Machine Value Type.
SDNode * getNode() const
getNode - Return the representative SDNode for this SUnit.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
const MCPhysReg * ImplicitDefs
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
MCRegAliasIterator enumerates all registers aliasing Reg.
This class provides iterator support for SDUse operands that use a specific SDNode.
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specific constraint if it is set.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
An unknown scheduling barrier.
static MVT getPhysicalRegisterVT(SDNode *N, unsigned Reg, const TargetInstrInfo *TII)
getPhysicalRegisterVT - Returns the ValueType of the physical register definition of the specified no...
void dump() const
Dump this node, for debugging.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
SDNode * getGluedUser() const
If this node has a glue value with a user, return the user (there is at most one).
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
const TargetRegisterClass * CopySrcRC
static SDNode * findGluedUser(SDNode *N)
findGluedUser - Find the representative use of a glue value by walking the use chain.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static RegisterScheduler fastDAGScheduler("fast","Fast suboptimal list scheduling", createFastDAGScheduler)
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
static use_iterator use_end()
iterator_range< value_op_iterator > op_values() const
void setLatency(unsigned Lat)
setLatency - Set the latency for this edge.
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Sequence
A sequence of states that a pointer may go through in which an objc_retain and objc_release are actua...
static bool CheckForLiveRegDef(SUnit *SU, unsigned Reg, std::vector< SUnit * > &LiveRegDefs, SmallSet< unsigned, 4 > &RegAdded, SmallVectorImpl< unsigned > &LRegs, const TargetRegisterInfo *TRI)
CheckForLiveRegDef - Return true and update live register vector if the specified register def of the...
static RegisterScheduler linearizeDAGScheduler("linearize","Linearize DAG, no scheduling", createDAGLinearizer)
int getNodeId() const
Return the unique node id.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
EVT getValueType() const
Return the ValueType of the referenced return value.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool addPred(const SDep &D, bool Required=true)
addPred - This adds the specified edge as a pred of the current node if not already.
SmallVector< SDep, 4 > Succs
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Arbitrary strong DAG edge (no real dependence).
ScheduleDAGSDNodes * createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createFastDAGScheduler - This creates a "fast" scheduler.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
void dump(const ScheduleDAG *G) const
SUnit - Scheduling unit.
SUnit - Scheduling unit. This is a node in the scheduling DAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.