38 #define DEBUG_TYPE "pre-RA-sched"
40 STATISTIC(NumBacktracks,
"Number of times scheduler backtracked");
41 STATISTIC(NumUnfolds,
"Number of nodes unfolded");
42 STATISTIC(NumDups,
"Number of duplicated nodes");
43 STATISTIC(NumPRCopies,
"Number of physical register copies");
47 "Bottom-up register reduction list scheduling",
51 "Similar to list-burr but schedules in source "
52 "order when possible",
57 "Bottom-up register pressure aware list scheduling "
58 "which tries to balance latency and register pressure",
63 "Bottom-up register pressure aware list scheduling "
64 "which tries to balance ILP and register pressure",
69 cl::desc(
"Disable cycle-level precision during preRA scheduling"));
75 cl::desc(
"Disable regpressure priority in sched=list-ilp"));
78 cl::desc(
"Disable live use priority in sched=list-ilp"));
81 cl::desc(
"Disable virtual register cycle interference checks"));
84 cl::desc(
"Disable physreg def-use affinity"));
87 cl::desc(
"Disable no-stall priority in sched=list-ilp"));
90 cl::desc(
"Disable critical path priority in sched=list-ilp"));
93 cl::desc(
"Disable scheduled-height priority in sched=list-ilp"));
96 cl::desc(
"Disable scheduler's two-address hack"));
100 cl::desc(
"Number of instructions to allow ahead of the critical path "
101 "in sched=list-ilp"));
105 cl::desc(
"Average inst/cycle whan no target itinerary exists."));
125 std::vector<SUnit*> PendingQueue;
134 unsigned MinAvailableCycle;
143 unsigned NumLiveRegs;
144 std::unique_ptr<SUnit*[]> LiveRegDefs;
145 std::unique_ptr<SUnit*[]> LiveRegGens;
166 NeedLatency(needlatency), AvailableQueue(availqueue), CurCycle(0),
167 Topo(SUnits, nullptr) {
176 ~ScheduleDAGRRList()
override {
178 delete AvailableQueue;
181 void Schedule()
override;
186 bool IsReachable(
const SUnit *SU,
const SUnit *TargetSU) {
187 return Topo.IsReachable(SU, TargetSU);
192 bool WillCreateCycle(
SUnit *SU,
SUnit *TargetSU) {
193 return Topo.WillCreateCycle(SU, TargetSU);
207 void RemovePred(
SUnit *SU,
const SDep &D) {
213 bool isReady(
SUnit *SU) {
215 AvailableQueue->isReady(SU);
218 void ReleasePred(
SUnit *SU,
const SDep *PredEdge);
219 void ReleasePredecessors(
SUnit *SU);
220 void ReleasePending();
221 void AdvanceToCycle(
unsigned NextCycle);
222 void AdvancePastStalls(
SUnit *SU);
223 void EmitNode(
SUnit *SU);
224 void ScheduleNodeBottomUp(
SUnit*);
225 void CapturePred(
SDep *PredEdge);
226 void UnscheduleNodeBottomUp(
SUnit*);
227 void RestoreHazardCheckerBottomUp();
230 void InsertCopiesAndMoveSuccs(
SUnit*,
unsigned,
236 void releaseInterferences(
unsigned Reg = 0);
238 SUnit *PickNodeToScheduleBottomUp();
239 void ListScheduleBottomUp();
244 unsigned NumSUnits = SUnits.size();
245 SUnit *NewNode = newSUnit(N);
247 if (NewNode->
NodeNum >= NumSUnits)
248 Topo.InitDAGTopologicalSorting();
255 unsigned NumSUnits = SUnits.size();
256 SUnit *NewNode = Clone(N);
258 if (NewNode->
NodeNum >= NumSUnits)
259 Topo.InitDAGTopologicalSorting();
265 bool forceUnitLatencies()
const override {
279 unsigned &RegClass,
unsigned &Cost,
292 RegClass = RC->
getID();
298 if (Opcode == TargetOpcode::REG_SEQUENCE) {
299 unsigned DstRCIdx = cast<ConstantSDNode>(Node->
getOperand(0))->getZExtValue();
301 RegClass = RC->
getID();
306 unsigned Idx = RegDefPos.
GetIdx();
309 RegClass = RC->
getID();
320 void ScheduleDAGRRList::Schedule() {
322 <<
"********** List Scheduling BB#" << BB->getNumber()
323 <<
" '" << BB->getName() <<
"' **********\n");
331 LiveRegDefs.reset(
new SUnit*[TRI->getNumRegs() + 1]());
332 LiveRegGens.reset(
new SUnit*[TRI->getNumRegs() + 1]());
333 CallSeqEndForStart.clear();
334 assert(Interferences.empty() && LRegsMap.empty() &&
"stale Interferences");
337 BuildSchedGraph(
nullptr);
341 Topo.InitDAGTopologicalSorting();
343 AvailableQueue->initNodes(SUnits);
348 ListScheduleBottomUp();
350 AvailableQueue->releaseState();
353 dbgs() <<
"*** Final schedule ***\n";
365 void ScheduleDAGRRList::ReleasePred(
SUnit *SU,
const SDep *PredEdge) {
370 dbgs() <<
"*** Scheduling failed! ***\n";
372 dbgs() <<
" has been released too many times!\n";
378 if (!forceUnitLatencies()) {
390 if (Height < MinAvailableCycle)
391 MinAvailableCycle = Height;
393 if (isReady(PredSU)) {
394 AvailableQueue->push(PredSU);
400 PendingQueue.push_back(PredSU);
439 goto found_chain_operand;
442 found_chain_operand:;
466 unsigned BestMaxNest = MaxNest;
468 unsigned MyNestLevel = NestLevel;
469 unsigned MyMaxNest = MaxNest;
471 MyNestLevel, MyMaxNest,
TII))
472 if (!Best || (MyMaxNest > BestMaxNest)) {
474 BestMaxNest = MyMaxNest;
478 MaxNest = BestMaxNest;
486 MaxNest = std::max(MaxNest, NestLevel);
499 goto found_chain_operand;
502 found_chain_operand:;
525 void ScheduleDAGRRList::ReleasePredecessors(
SUnit *SU) {
529 ReleasePred(SU, &*
I);
530 if (
I->isAssignedRegDep()) {
535 SUnit *RegDef = LiveRegDefs[
I->getReg()]; (void)RegDef;
536 assert((!RegDef || RegDef == SU || RegDef ==
I->getSUnit()) &&
537 "interference on register dependence");
538 LiveRegDefs[
I->getReg()] =
I->getSUnit();
539 if (!LiveRegGens[
I->getReg()]) {
541 LiveRegGens[
I->getReg()] = SU;
549 unsigned CallResource = TRI->getNumRegs();
550 if (!LiveRegDefs[CallResource])
552 if (Node->isMachineOpcode() &&
553 Node->getMachineOpcode() == (
unsigned)
TII->getCallFrameDestroyOpcode()) {
554 unsigned NestLevel = 0;
555 unsigned MaxNest = 0;
559 CallSeqEndForStart[
Def] = SU;
562 LiveRegDefs[CallResource] =
Def;
563 LiveRegGens[CallResource] = SU;
570 void ScheduleDAGRRList::ReleasePending() {
572 assert(PendingQueue.empty() &&
"pending instrs not allowed in this mode");
577 if (AvailableQueue->empty())
578 MinAvailableCycle = UINT_MAX;
582 for (
unsigned i = 0, e = PendingQueue.size();
i != e; ++
i) {
583 unsigned ReadyCycle = PendingQueue[
i]->getHeight();
584 if (ReadyCycle < MinAvailableCycle)
585 MinAvailableCycle = ReadyCycle;
588 if (!isReady(PendingQueue[
i]))
590 AvailableQueue->push(PendingQueue[i]);
592 PendingQueue[
i]->isPending =
false;
593 PendingQueue[
i] = PendingQueue.back();
594 PendingQueue.pop_back();
600 void ScheduleDAGRRList::AdvanceToCycle(
unsigned NextCycle) {
601 if (NextCycle <= CurCycle)
605 AvailableQueue->setCurCycle(NextCycle);
606 if (!HazardRec->isEnabled()) {
608 CurCycle = NextCycle;
611 for (; CurCycle != NextCycle; ++CurCycle) {
612 HazardRec->RecedeCycle();
622 void ScheduleDAGRRList::AdvancePastStalls(
SUnit *SU) {
639 AdvanceToCycle(ReadyCycle);
652 HazardRec->getHazardType(SU, -Stalls);
659 AdvanceToCycle(CurCycle + Stalls);
664 void ScheduleDAGRRList::EmitNode(
SUnit *SU) {
665 if (!HazardRec->isEnabled())
675 "This target-independent node should not be scheduled.");
698 HazardRec->EmitInstruction(SU);
706 void ScheduleDAGRRList::ScheduleNodeBottomUp(
SUnit *SU) {
707 DEBUG(
dbgs() <<
"\n*** Scheduling [" << CurCycle <<
"]: ");
711 if (CurCycle < SU->getHeight())
713 <<
"] pipeline stall!\n");
727 AvailableQueue->scheduledNode(SU);
732 if (!HazardRec->isEnabled() &&
AvgIPC < 2)
733 AdvanceToCycle(CurCycle + 1);
737 ReleasePredecessors(SU);
743 if (
I->isAssignedRegDep() && LiveRegDefs[
I->getReg()] == SU) {
744 assert(NumLiveRegs > 0 &&
"NumLiveRegs is already zero!");
746 LiveRegDefs[
I->getReg()] =
nullptr;
747 LiveRegGens[
I->getReg()] =
nullptr;
748 releaseInterferences(
I->getReg());
753 unsigned CallResource = TRI->getNumRegs();
754 if (LiveRegDefs[CallResource] == SU)
757 if (SUNode->isMachineOpcode() &&
758 SUNode->getMachineOpcode() == (
unsigned)
TII->getCallFrameSetupOpcode()) {
759 assert(NumLiveRegs > 0 &&
"NumLiveRegs is already zero!");
761 LiveRegDefs[CallResource] =
nullptr;
762 LiveRegGens[CallResource] =
nullptr;
763 releaseInterferences(CallResource);
779 if (HazardRec->isEnabled() ||
AvgIPC > 1) {
782 if ((HazardRec->isEnabled() && HazardRec->atIssueLimit())
783 || (!HazardRec->isEnabled() && IssueCount ==
AvgIPC))
784 AdvanceToCycle(CurCycle + 1);
791 void ScheduleDAGRRList::CapturePred(
SDep *PredEdge) {
796 AvailableQueue->remove(PredSU);
805 void ScheduleDAGRRList::UnscheduleNodeBottomUp(
SUnit *SU) {
812 if (
I->isAssignedRegDep() && SU == LiveRegGens[
I->getReg()]){
813 assert(NumLiveRegs > 0 &&
"NumLiveRegs is already zero!");
814 assert(LiveRegDefs[
I->getReg()] ==
I->getSUnit() &&
815 "Physical register dependency violated?");
817 LiveRegDefs[
I->getReg()] =
nullptr;
818 LiveRegGens[
I->getReg()] =
nullptr;
819 releaseInterferences(
I->getReg());
825 unsigned CallResource = TRI->getNumRegs();
828 if (SUNode->isMachineOpcode() &&
829 SUNode->getMachineOpcode() == (
unsigned)
TII->getCallFrameSetupOpcode()) {
831 LiveRegDefs[CallResource] = SU;
832 LiveRegGens[CallResource] = CallSeqEndForStart[SU];
838 if (LiveRegGens[CallResource] == SU)
841 if (SUNode->isMachineOpcode() &&
842 SUNode->getMachineOpcode() == (
unsigned)
TII->getCallFrameDestroyOpcode()) {
843 assert(NumLiveRegs > 0 &&
"NumLiveRegs is already zero!");
845 LiveRegDefs[CallResource] =
nullptr;
846 LiveRegGens[CallResource] =
nullptr;
847 releaseInterferences(CallResource);
851 for (
auto &Succ : SU->
Succs) {
852 if (Succ.isAssignedRegDep()) {
853 auto Reg = Succ.getReg();
854 if (!LiveRegDefs[
Reg])
858 LiveRegDefs[
Reg] = SU;
862 if (!LiveRegGens[Reg]) {
864 LiveRegGens[
Reg] = Succ.getSUnit();
865 for (
auto &Succ2 : SU->
Succs) {
866 if (Succ2.isAssignedRegDep() && Succ2.getReg() == Reg &&
867 Succ2.getSUnit()->getHeight() < LiveRegGens[
Reg]->getHeight())
868 LiveRegGens[Reg] = Succ2.getSUnit();
882 PendingQueue.push_back(SU);
885 AvailableQueue->push(SU);
887 AvailableQueue->unscheduledNode(SU);
892 void ScheduleDAGRRList::RestoreHazardCheckerBottomUp() {
896 HazardRec->getMaxLookAhead());
900 std::vector<SUnit*>::const_iterator
I = (
Sequence.end() - LookAhead);
901 unsigned HazardCycle = (*I)->getHeight();
902 for (std::vector<SUnit*>::const_iterator
E =
Sequence.end(); I !=
E; ++
I) {
904 for (; SU->
getHeight() > HazardCycle; ++HazardCycle) {
905 HazardRec->RecedeCycle();
913 void ScheduleDAGRRList::BacktrackBottomUp(
SUnit *SU,
SUnit *BtSU) {
919 UnscheduleNodeBottomUp(OldSU);
920 AvailableQueue->setCurCycle(CurCycle);
928 RestoreHazardCheckerBottomUp();
938 if (SUNode->isOperandOf(N))
946 SUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(
SUnit *SU) {
955 bool TryUnfold =
false;
964 MVT VT =
Op.getNode()->getSimpleValueType(
Op.getResNo());
971 if (!
TII->unfoldMemoryOperand(*DAG, N, NewNodes))
976 if (NewNodes.
size() == 3)
980 assert(NewNodes.
size() == 2 &&
"Expected a load folding node!");
983 SDNode *LoadNode = NewNodes[0];
984 unsigned NumVals = N->getNumValues();
986 for (
unsigned i = 0; i != NumVals; ++
i)
988 DAG->ReplaceAllUsesOfValueWith(
SDValue(SU->
getNode(), OldNumVals-1),
994 bool isNewLoad =
true;
1000 LoadSU = CreateNewSUnit(LoadNode);
1003 InitNumRegDefsLeft(LoadSU);
1004 computeLatency(LoadSU);
1007 SUnit *NewSU = CreateNewSUnit(N);
1008 assert(N->getNodeId() == -1 &&
"Node already inserted!");
1021 InitNumRegDefsLeft(NewSU);
1022 computeLatency(NewSU);
1046 for (
const SDep &Pred : ChainPreds) {
1047 RemovePred(SU, Pred);
1049 AddPred(LoadSU, Pred);
1051 for (
const SDep &Pred : LoadPreds) {
1052 RemovePred(SU, Pred);
1054 AddPred(LoadSU, Pred);
1056 for (
const SDep &Pred : NodePreds) {
1057 RemovePred(SU, Pred);
1058 AddPred(NewSU, Pred);
1060 for (
SDep D : NodeSuccs) {
1063 RemovePred(SuccDep, D);
1065 AddPred(SuccDep, D);
1067 if (AvailableQueue->tracksRegPressure() && SuccDep->
isScheduled
1071 for (
SDep D : ChainSuccs) {
1074 RemovePred(SuccDep, D);
1077 AddPred(SuccDep, D);
1088 AvailableQueue->addNode(LoadSU);
1089 AvailableQueue->addNode(NewSU);
1101 NewSU = CreateClone(SU);
1106 AddPred(NewSU, Pred);
1120 DelDeps.
push_back(std::make_pair(SuccSU, D));
1123 for (
auto &DelDep : DelDeps)
1124 RemovePred(DelDep.first, DelDep.second);
1126 AvailableQueue->updateNode(SU);
1127 AvailableQueue->addNode(NewSU);
1135 void ScheduleDAGRRList::InsertCopiesAndMoveSuccs(
SUnit *SU,
unsigned Reg,
1139 SUnit *CopyFromSU = CreateNewSUnit(
nullptr);
1143 SUnit *CopyToSU = CreateNewSUnit(
nullptr);
1158 DelDeps.
push_back(std::make_pair(SuccSU, Succ));
1167 for (
auto &DelDep : DelDeps)
1168 RemovePred(DelDep.first, DelDep.second);
1171 FromDep.setLatency(SU->
Latency);
1172 AddPred(CopyFromSU, FromDep);
1174 ToDep.setLatency(CopyFromSU->
Latency);
1175 AddPred(CopyToSU, ToDep);
1177 AvailableQueue->updateNode(SU);
1178 AvailableQueue->addNode(CopyFromSU);
1179 AvailableQueue->addNode(CopyToSU);
1211 SUnit **LiveRegDefs,
1218 if (!LiveRegDefs[*AliasI])
continue;
1221 if (LiveRegDefs[*AliasI] == SU)
continue;
1224 if (RegAdded.
insert(*AliasI).second) {
1237 for (
unsigned i = 1, e = LiveRegDefs.
size()-1; i != e; ++
i) {
1238 if (!LiveRegDefs[i])
continue;
1239 if (LiveRegDefs[i] == SU)
continue;
1241 if (RegAdded.
insert(i).second)
1249 if (
const auto *RegOp = dyn_cast<RegisterMaskSDNode>(
Op.getNode()))
1250 return RegOp->getRegMask();
1258 bool ScheduleDAGRRList::
1260 if (NumLiveRegs == 0)
1270 if (I->isAssignedRegDep() && LiveRegDefs[I->getReg()] != SU)
1272 RegAdded, LRegs, TRI);
1278 unsigned NumOps = Node->getNumOperands();
1279 if (Node->getOperand(NumOps-1).getValueType() ==
MVT::Glue)
1284 cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
1292 for (; NumVals; --NumVals, ++
i) {
1293 unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->
getReg();
1303 if (!Node->isMachineOpcode())
1308 if (Node->getMachineOpcode() == (
unsigned)
TII->getCallFrameDestroyOpcode()) {
1310 unsigned CallResource = TRI->getNumRegs();
1311 if (LiveRegDefs[CallResource]) {
1312 SDNode *Gen = LiveRegGens[CallResource]->getNode();
1316 RegAdded.
insert(CallResource).second)
1332 return !LRegs.
empty();
1335 void ScheduleDAGRRList::releaseInterferences(
unsigned Reg) {
1337 for (
unsigned i = Interferences.size(); i > 0; --
i) {
1338 SUnit *SU = Interferences[i-1];
1339 LRegsMapT::iterator LRegsPos = LRegsMap.find(SU);
1351 AvailableQueue->push(SU);
1353 if (i < Interferences.size())
1354 Interferences[i-1] = Interferences.back();
1355 Interferences.pop_back();
1356 LRegsMap.erase(LRegsPos);
1364 SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() {
1365 SUnit *CurSU = AvailableQueue->empty() ?
nullptr : AvailableQueue->pop();
1368 if (!DelayForLiveRegsBottomUp(CurSU, LRegs))
1371 (LRegs[0] == TRI->getNumRegs() ?
"CallResource"
1372 : TRI->getName(LRegs[0]))
1373 <<
" SU #" << CurSU->
NodeNum <<
'\n');
1374 std::pair<LRegsMapT::iterator, bool> LRegsPair =
1375 LRegsMap.insert(std::make_pair(CurSU, LRegs));
1376 if (LRegsPair.second) {
1378 Interferences.push_back(CurSU);
1383 LRegsPair.first->second = LRegs;
1385 CurSU = AvailableQueue->pop();
1393 for (
SUnit *TrySU : Interferences) {
1398 SUnit *BtSU =
nullptr;
1399 unsigned LiveCycle = UINT_MAX;
1400 for (
unsigned Reg : LRegs) {
1401 if (LiveRegGens[Reg]->getHeight() < LiveCycle) {
1402 BtSU = LiveRegGens[
Reg];
1406 if (!WillCreateCycle(TrySU, BtSU)) {
1408 BacktrackBottomUp(TrySU, BtSU);
1415 AvailableQueue->remove(BtSU);
1418 << TrySU->NodeNum <<
")\n");
1423 if (!TrySU->isAvailable || !TrySU->NodeQueueId)
1424 CurSU = AvailableQueue->pop();
1427 AvailableQueue->remove(TrySU);
1441 SUnit *TrySU = Interferences[0];
1443 assert(LRegs.
size() == 1 &&
"Can't handle this yet!");
1444 unsigned Reg = LRegs[0];
1448 TRI->getMinimalPhysRegClass(Reg, VT);
1458 SUnit *NewDef =
nullptr;
1460 NewDef = CopyAndMoveSuccessors(LRDef);
1461 if (!DestRC && !NewDef)
1467 InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
1471 NewDef = Copies.
back();
1475 <<
" to SU #" << TrySU->
NodeNum <<
"\n");
1476 LiveRegDefs[
Reg] = NewDef;
1481 assert(CurSU &&
"Unable to resolve live physical register dependencies!");
1487 void ScheduleDAGRRList::ListScheduleBottomUp() {
1489 ReleasePredecessors(&ExitSU);
1492 if (!SUnits.empty()) {
1493 SUnit *RootSU = &SUnits[DAG->getRoot().getNode()->getNodeId()];
1494 assert(RootSU->
Succs.empty() &&
"Graph root shouldn't have successors!");
1496 AvailableQueue->push(RootSU);
1502 while (!AvailableQueue->empty() || !Interferences.empty()) {
1503 DEBUG(
dbgs() <<
"\nExamining Available:\n";
1504 AvailableQueue->dump(
this));
1508 SUnit *SU = PickNodeToScheduleBottomUp();
1510 AdvancePastStalls(SU);
1512 ScheduleNodeBottomUp(SU);
1514 while (AvailableQueue->empty() && !PendingQueue.empty()) {
1516 assert(MinAvailableCycle < UINT_MAX &&
"MinAvailableCycle uninitialized");
1517 AdvanceToCycle(std::max(CurCycle + 1, MinAvailableCycle));
1525 VerifyScheduledSequence(
true);
1537 class RegReductionPQBase;
1539 struct queue_sort :
public std::binary_function<SUnit*, SUnit*, bool> {
1540 bool isReady(
SUnit* SU,
unsigned CurCycle)
const {
return true; }
1545 struct reverse_sort :
public queue_sort {
1547 reverse_sort(SF &sf) : SortFunc(sf) {}
1549 bool operator()(
SUnit* left,
SUnit* right)
const {
1552 return SortFunc(right, left);
1559 struct bu_ls_rr_sort :
public queue_sort {
1562 HasReadyFilter =
false
1565 RegReductionPQBase *SPQ;
1566 bu_ls_rr_sort(RegReductionPQBase *spq) : SPQ(spq) {}
1568 bool operator()(
SUnit* left,
SUnit* right)
const;
1572 struct src_ls_rr_sort :
public queue_sort {
1575 HasReadyFilter =
false
1578 RegReductionPQBase *SPQ;
1579 src_ls_rr_sort(RegReductionPQBase *spq)
1582 bool operator()(
SUnit* left,
SUnit* right)
const;
1586 struct hybrid_ls_rr_sort :
public queue_sort {
1589 HasReadyFilter =
false
1592 RegReductionPQBase *SPQ;
1593 hybrid_ls_rr_sort(RegReductionPQBase *spq)
1596 bool isReady(
SUnit *SU,
unsigned CurCycle)
const;
1598 bool operator()(
SUnit* left,
SUnit* right)
const;
1603 struct ilp_ls_rr_sort :
public queue_sort {
1606 HasReadyFilter =
false
1609 RegReductionPQBase *SPQ;
1610 ilp_ls_rr_sort(RegReductionPQBase *spq)
1613 bool isReady(
SUnit *SU,
unsigned CurCycle)
const;
1615 bool operator()(
SUnit* left,
SUnit* right)
const;
1620 std::vector<SUnit*>
Queue;
1621 unsigned CurQueueId;
1622 bool TracksRegPressure;
1626 std::vector<SUnit> *SUnits;
1632 ScheduleDAGRRList *scheduleDAG;
1635 std::vector<unsigned> SethiUllmanNumbers;
1643 std::vector<unsigned> RegLimit;
1647 bool hasReadyFilter,
1654 CurQueueId(0), TracksRegPressure(tracksrp), SrcOrder(srcorder),
1655 MF(mf),
TII(tii), TRI(tri), TLI(tli), scheduleDAG(nullptr) {
1656 if (TracksRegPressure) {
1657 unsigned NumRC = TRI->getNumRegClasses();
1658 RegLimit.resize(NumRC);
1660 std::fill(RegLimit.begin(), RegLimit.end(), 0);
1663 E = TRI->regclass_end(); I !=
E; ++
I)
1668 void setScheduleDAG(ScheduleDAGRRList *scheduleDag) {
1669 scheduleDAG = scheduleDag;
1673 return scheduleDAG->getHazardRec();
1676 void initNodes(std::vector<SUnit> &sunits)
override;
1678 void addNode(
const SUnit *SU)
override;
1680 void updateNode(
const SUnit *SU)
override;
1682 void releaseState()
override {
1684 SethiUllmanNumbers.clear();
1688 unsigned getNodePriority(
const SUnit *SU)
const;
1690 unsigned getNodeOrdering(
const SUnit *SU)
const {
1696 bool empty()
const override {
return Queue.empty(); }
1698 void push(
SUnit *U)
override {
1704 void remove(
SUnit *SU)
override {
1707 std::vector<SUnit *>::iterator I =
find(
Queue, SU);
1708 if (I != std::prev(
Queue.end()))
1714 bool tracksRegPressure()
const override {
return TracksRegPressure; }
1716 void dumpRegPressure()
const;
1718 bool HighRegPressure(
const SUnit *SU)
const;
1720 bool MayReduceRegPressure(
SUnit *SU)
const;
1722 int RegPressureDiff(
SUnit *SU,
unsigned &LiveUses)
const;
1724 void scheduledNode(
SUnit *SU)
override;
1726 void unscheduledNode(
SUnit *SU)
override;
1730 void AddPseudoTwoAddrDeps();
1731 void PrescheduleNodesWithMultipleUses();
1732 void CalculateSethiUllmanNumbers();
1736 static SUnit *popFromQueueImpl(std::vector<SUnit*> &Q, SF &Picker) {
1737 std::vector<SUnit *>::iterator Best = Q.begin();
1738 for (std::vector<SUnit *>::iterator I = std::next(Q.begin()),
1739 E = Q.end(); I !=
E; ++
I)
1740 if (Picker(*Best, *I))
1743 if (Best != std::prev(Q.end()))
1750 SUnit *popFromQueue(std::vector<SUnit*> &Q, SF &Picker,
ScheduleDAG *DAG) {
1753 reverse_sort<SF> RPicker(Picker);
1754 return popFromQueueImpl(Q, RPicker);
1758 return popFromQueueImpl(Q, Picker);
1762 class RegReductionPriorityQueue :
public RegReductionPQBase {
1772 : RegReductionPQBase(mf, SF::HasReadyFilter, tracksrp, srcorder,
1776 bool isBottomUp()
const override {
return SF::IsBottomUp; }
1778 bool isReady(
SUnit *U)
const override {
1779 return Picker.HasReadyFilter && Picker.isReady(U, getCurCycle());
1782 SUnit *pop()
override {
1783 if (
Queue.empty())
return nullptr;
1785 SUnit *V = popFromQueue(
Queue, Picker, scheduleDAG);
1790 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1793 std::vector<SUnit*> DumpQueue =
Queue;
1794 SF DumpPicker = Picker;
1795 while (!DumpQueue.empty()) {
1796 SUnit *SU = popFromQueue(DumpQueue, DumpPicker, scheduleDAG);
1804 typedef RegReductionPriorityQueue<bu_ls_rr_sort>
1805 BURegReductionPriorityQueue;
1807 typedef RegReductionPriorityQueue<src_ls_rr_sort>
1808 SrcRegReductionPriorityQueue;
1810 typedef RegReductionPriorityQueue<hybrid_ls_rr_sort>
1811 HybridBURRPriorityQueue;
1813 typedef RegReductionPriorityQueue<ilp_ls_rr_sort>
1814 ILPBURRPriorityQueue;
1830 if (LSchedLow != RSchedLow)
1831 return LSchedLow < RSchedLow ? 1 : -1;
1839 unsigned &SethiUllmanNumber = SUNumbers[SU->
NodeNum];
1840 if (SethiUllmanNumber != 0)
1841 return SethiUllmanNumber;
1845 if (Pred.
isCtrl())
continue;
1848 if (PredSethiUllman > SethiUllmanNumber) {
1849 SethiUllmanNumber = PredSethiUllman;
1851 }
else if (PredSethiUllman == SethiUllmanNumber)
1855 SethiUllmanNumber += Extra;
1857 if (SethiUllmanNumber == 0)
1858 SethiUllmanNumber = 1;
1860 return SethiUllmanNumber;
1865 void RegReductionPQBase::CalculateSethiUllmanNumbers() {
1866 SethiUllmanNumbers.assign(SUnits->size(), 0);
1868 for (
const SUnit &SU : *SUnits)
1872 void RegReductionPQBase::addNode(
const SUnit *SU) {
1873 unsigned SUSize = SethiUllmanNumbers.size();
1874 if (SUnits->size() > SUSize)
1875 SethiUllmanNumbers.resize(SUSize*2, 0);
1879 void RegReductionPQBase::updateNode(
const SUnit *SU) {
1880 SethiUllmanNumbers[SU->
NodeNum] = 0;
1886 unsigned RegReductionPQBase::getNodePriority(
const SUnit *SU)
const {
1893 if (Opc == TargetOpcode::EXTRACT_SUBREG ||
1894 Opc == TargetOpcode::SUBREG_TO_REG ||
1895 Opc == TargetOpcode::INSERT_SUBREG)
1911 return SethiUllmanNumbers[SU->
NodeNum];
1913 unsigned Priority = SethiUllmanNumbers[SU->
NodeNum];
1917 return (NP > 0) ? NP : 0;
1927 void RegReductionPQBase::dumpRegPressure()
const {
1928 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1930 E = TRI->regclass_end(); I !=
E; ++
I) {
1935 DEBUG(
dbgs() << TRI->getRegClassName(RC) <<
": " << RP <<
" / "
1936 << RegLimit[
Id] <<
'\n');
1941 bool RegReductionPQBase::HighRegPressure(
const SUnit *SU)
const {
1955 RegDefPos.
IsValid(); RegDefPos.Advance()) {
1956 unsigned RCId, Cost;
1966 bool RegReductionPQBase::MayReduceRegPressure(
SUnit *SU)
const {
1973 for (
unsigned i = 0; i != NumDefs; ++
i) {
1977 unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
1991 int RegReductionPQBase::RegPressureDiff(
SUnit *SU,
unsigned &LiveUses)
const {
2006 RegDefPos.
IsValid(); RegDefPos.Advance()) {
2007 MVT VT = RegDefPos.GetValue();
2008 unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
2019 for (
unsigned i = 0; i != NumDefs; ++
i) {
2023 unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
2030 void RegReductionPQBase::scheduledNode(
SUnit *SU) {
2031 if (!TracksRegPressure)
2064 RegDefPos.
IsValid(); RegDefPos.Advance(), --SkipRegDefs) {
2068 unsigned RCId, Cost;
2080 RegDefPos.
IsValid(); RegDefPos.Advance(), --SkipRegDefs) {
2081 if (SkipRegDefs > 0)
2083 unsigned RCId, Cost;
2098 void RegReductionPQBase::unscheduledNode(
SUnit *SU) {
2099 if (!TracksRegPressure)
2110 if (Opc == TargetOpcode::EXTRACT_SUBREG ||
2111 Opc == TargetOpcode::INSERT_SUBREG ||
2112 Opc == TargetOpcode::SUBREG_TO_REG ||
2113 Opc == TargetOpcode::REG_SEQUENCE ||
2114 Opc == TargetOpcode::IMPLICIT_DEF)
2130 unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
2131 RegPressure[RCId] += TLI->getRepRegClassCostFor(VT);
2136 if (POpc == TargetOpcode::IMPLICIT_DEF)
2138 if (POpc == TargetOpcode::EXTRACT_SUBREG ||
2139 POpc == TargetOpcode::INSERT_SUBREG ||
2140 POpc == TargetOpcode::SUBREG_TO_REG) {
2142 unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
2143 RegPressure[RCId] += TLI->getRepRegClassCostFor(VT);
2147 for (
unsigned i = 0; i != NumDefs; ++
i) {
2151 unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
2152 if (
RegPressure[RCId] < TLI->getRepRegClassCostFor(VT))
2156 RegPressure[RCId] -= TLI->getRepRegClassCostFor(VT);
2164 for (
unsigned i = NumDefs, e = N->
getNumValues(); i != e; ++
i) {
2170 unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
2171 RegPressure[RCId] += TLI->getRepRegClassCostFor(VT);
2185 unsigned MaxHeight = 0;
2187 if (Succ.
isCtrl())
continue;
2194 if (Height > MaxHeight)
2203 unsigned Scratches = 0;
2205 if (Pred.
isCtrl())
continue;
2214 bool RetVal =
false;
2216 if (Pred.
isCtrl())
continue;
2236 bool RetVal =
false;
2238 if (Succ.
isCtrl())
continue;
2275 if (Pred.
isCtrl())
continue;
2287 if (Pred.
isCtrl())
continue;
2291 "VRegCycle def must be CopyFromReg");
2305 if (Pred.
isCtrl())
continue;
2319 if ((
int)SPQ->getCurCycle() < Height)
return true;
2320 if (SPQ->getHazardRec()->getHazardType(SU, 0)
2329 RegReductionPQBase *SPQ) {
2334 int LHeight = (int)left->
getHeight() + LPenalty;
2335 int RHeight = (int)right->
getHeight() + RPenalty;
2348 if (LHeight != RHeight)
2349 return LHeight > RHeight ? 1 : -1;
2361 if (!SPQ->getHazardRec()->isEnabled()) {
2362 if (LHeight != RHeight)
2363 return LHeight > RHeight ? 1 : -1;
2365 int LDepth = left->
getDepth() - LPenalty;
2366 int RDepth = right->
getDepth() - RPenalty;
2367 if (LDepth != RDepth) {
2369 <<
") depth " << LDepth <<
" vs SU (" << right->
NodeNum
2370 <<
") depth " << RDepth <<
"\n");
2371 return LDepth < RDepth ? 1 : -1;
2387 if (LHasPhysReg != RHasPhysReg) {
2389 static const char *
const PhysRegMsg[] = {
" has no physreg",
2390 " defines a physreg" };
2393 << PhysRegMsg[LHasPhysReg] <<
" SU(" << right->
NodeNum <<
") "
2394 << PhysRegMsg[RHasPhysReg] <<
"\n");
2395 return LHasPhysReg < RHasPhysReg;
2400 unsigned LPriority = SPQ->getNodePriority(left);
2401 unsigned RPriority = SPQ->getNodePriority(right);
2407 RPriority = (RPriority > RNumVals) ? (RPriority - RNumVals) : 0;
2411 LPriority = (LPriority > LNumVals) ? (LPriority - LNumVals) : 0;
2414 if (LPriority != RPriority)
2415 return LPriority > RPriority;
2420 unsigned LOrder = SPQ->getNodeOrdering(left);
2421 unsigned ROrder = SPQ->getNodeOrdering(right);
2425 if ((LOrder || ROrder) && LOrder != ROrder)
2426 return LOrder != 0 && (LOrder < ROrder || ROrder == 0);
2449 return LDist < RDist;
2454 if (LScratch != RScratch)
2455 return LScratch > RScratch;
2459 if ((left->
isCall && RPriority > 0) || (right->
isCall && LPriority > 0))
2478 "NodeQueueId cannot be zero");
2483 bool bu_ls_rr_sort::operator()(
SUnit *left,
SUnit *right)
const {
2491 bool src_ls_rr_sort::operator()(
SUnit *left,
SUnit *right)
const {
2495 unsigned LOrder = SPQ->getNodeOrdering(left);
2496 unsigned ROrder = SPQ->getNodeOrdering(right);
2500 if ((LOrder || ROrder) && LOrder != ROrder)
2501 return LOrder != 0 && (LOrder < ROrder || ROrder == 0);
2510 bool hybrid_ls_rr_sort::isReady(
SUnit *SU,
unsigned CurCycle)
const {
2511 static const unsigned ReadyDelay = 3;
2513 if (SPQ->MayReduceRegPressure(SU))
return true;
2515 if (SU->
getHeight() > (CurCycle + ReadyDelay))
return false;
2517 if (SPQ->getHazardRec()->getHazardType(SU, -ReadyDelay)
2525 bool hybrid_ls_rr_sort::operator()(
SUnit *left,
SUnit *right)
const {
2533 bool LHigh = SPQ->HighRegPressure(left);
2534 bool RHigh = SPQ->HighRegPressure(right);
2537 if (LHigh && !RHigh) {
2542 else if (!LHigh && RHigh) {
2547 if (!LHigh && !RHigh) {
2557 bool ilp_ls_rr_sort::isReady(
SUnit *SU,
unsigned CurCycle)
const {
2558 if (SU->
getHeight() > CurCycle)
return false;
2560 if (SPQ->getHazardRec()->getHazardType(SU, 0)
2574 if (Opc == TargetOpcode::EXTRACT_SUBREG ||
2575 Opc == TargetOpcode::SUBREG_TO_REG ||
2576 Opc == TargetOpcode::INSERT_SUBREG)
2591 bool ilp_ls_rr_sort::operator()(
SUnit *left,
SUnit *right)
const {
2599 unsigned LLiveUses = 0, RLiveUses = 0;
2600 int LPDiff = 0, RPDiff = 0;
2602 LPDiff = SPQ->RegPressureDiff(left, LLiveUses);
2603 RPDiff = SPQ->RegPressureDiff(right, RLiveUses);
2607 <<
" != SU(" << right->
NodeNum <<
"): " << RPDiff <<
"\n");
2608 return LPDiff > RPDiff;
2614 if (LReduce && !RReduce)
return false;
2615 if (RReduce && !LReduce)
return true;
2620 <<
" != SU(" << right->
NodeNum <<
"): " << RLiveUses <<
"\n");
2621 return LLiveUses < RLiveUses;
2627 if (LStall != RStall)
2650 void RegReductionPQBase::initNodes(std::vector<SUnit> &sunits) {
2654 AddPseudoTwoAddrDeps();
2656 if (!TracksRegPressure && !SrcOrder)
2657 PrescheduleNodesWithMultipleUses();
2659 CalculateSethiUllmanNumbers();
2662 if (scheduleDAG->BB->isSuccessor(scheduleDAG->BB))
2663 for (
SUnit &SU : sunits)
2671 bool RegReductionPQBase::canClobber(
const SUnit *SU,
const SUnit *
Op) {
2677 for (
unsigned i = 0; i != NumOps; ++
i) {
2693 ScheduleDAGRRList *scheduleDAG,
2699 if(!ImpDefs && !RegMask)
2704 for (
const SDep &SuccPred : SuccSU->
Preds) {
2710 scheduleDAG->IsReachable(DepSU, SuccPred.
getSUnit()))
2714 for (
const MCPhysReg *ImpDef = ImpDefs; *ImpDef; ++ImpDef)
2719 scheduleDAG->IsReachable(DepSU, SuccPred.
getSUnit()))
2734 assert(ImpDefs &&
"Caller should check hasPhysRegDefs");
2737 if (!SUNode->isMachineOpcode())
2740 TII->
get(SUNode->getMachineOpcode()).getImplicitDefs();
2742 if (!SUImpDefs && !SURegMask)
2744 for (
unsigned i = NumDefs, e = N->
getNumValues(); i != e; ++
i) {
2750 unsigned Reg = ImpDefs[i - NumDefs];
2755 for (;*SUImpDefs; ++SUImpDefs) {
2756 unsigned SUReg = *SUImpDefs;
2796 void RegReductionPQBase::PrescheduleNodesWithMultipleUses() {
2798 for (
SUnit &SU : *SUnits) {
2816 SUnit *PredSU =
nullptr;
2840 for (
const SDep &PredSucc : PredSU->
Succs) {
2842 if (PredSuccSU == &SU)
continue;
2846 goto outer_loop_continue;
2850 goto outer_loop_continue;
2852 if (scheduleDAG->IsReachable(&SU, PredSuccSU))
2853 goto outer_loop_continue;
2859 <<
" next to PredSU #" << PredSU->
NodeNum
2860 <<
" to guide scheduling in the presence of multiple uses\n");
2861 for (
unsigned i = 0; i != PredSU->
Succs.size(); ++
i) {
2865 if (SuccSU != &SU) {
2867 scheduleDAG->RemovePred(SuccSU, Edge);
2868 scheduleDAG->AddPred(&SU, Edge);
2870 scheduleDAG->AddPred(SuccSU, Edge);
2874 outer_loop_continue:;
2885 void RegReductionPQBase::AddPseudoTwoAddrDeps() {
2886 for (
SUnit &SU : *SUnits) {
2899 for (
unsigned j = 0; j != NumOps; ++j) {
2923 while (SuccSU->
Succs.size() == 1 &&
2926 TargetOpcode::COPY_TO_REGCLASS)
2927 SuccSU = SuccSU->
Succs.front().getSUnit();
2940 if (SuccOpc == TargetOpcode::EXTRACT_SUBREG ||
2941 SuccOpc == TargetOpcode::INSERT_SUBREG ||
2942 SuccOpc == TargetOpcode::SUBREG_TO_REG)
2945 (!canClobber(SuccSU, DUSU) ||
2948 !scheduleDAG->IsReachable(SuccSU, &SU)) {
2949 DEBUG(
dbgs() <<
" Adding a pseudo-two-addr edge from SU #"
2969 BURegReductionPriorityQueue *PQ =
2970 new BURegReductionPriorityQueue(*IS->
MF,
false,
false, TII, TRI,
nullptr);
2971 ScheduleDAGRRList *SD =
new ScheduleDAGRRList(*IS->
MF,
false, PQ, OptLevel);
2972 PQ->setScheduleDAG(SD);
2983 SrcRegReductionPriorityQueue *PQ =
2984 new SrcRegReductionPriorityQueue(*IS->
MF,
false,
true, TII, TRI,
nullptr);
2985 ScheduleDAGRRList *SD =
new ScheduleDAGRRList(*IS->
MF,
false, PQ, OptLevel);
2986 PQ->setScheduleDAG(SD);
2998 HybridBURRPriorityQueue *PQ =
2999 new HybridBURRPriorityQueue(*IS->
MF,
true,
false, TII, TRI, TLI);
3001 ScheduleDAGRRList *SD =
new ScheduleDAGRRList(*IS->
MF,
true, PQ, OptLevel);
3002 PQ->setScheduleDAG(SD);
3014 ILPBURRPriorityQueue *PQ =
3015 new ILPBURRPriorityQueue(*IS->
MF,
true,
false, TII, TRI, TLI);
3016 ScheduleDAGRRList *SD =
new ScheduleDAGRRList(*IS->
MF,
true, PQ, OptLevel);
3017 PQ->setScheduleDAG(SD);
void push_back(const T &Elt)
bool isCtrl() const
isCtrl - Shorthand for getKind() != SDep::Data.
const SDNode * GetNode() const
bool isSucc(SUnit *N)
isSucc - Test if node N is a successor of this node.
STATISTIC(NumFunctions,"Total number of functions")
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
static int BUCompareLatency(SUnit *left, SUnit *right, bool checkPref, RegReductionPQBase *SPQ)
bool isCommutable() const
Return true if this may be a 2- or 3-address instruction (of the form "X = op Y, Z, ..."), which produces the same result if Y and Z are exchanged.
static bool canEnableCoalescing(SUnit *SU)
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
static cl::opt< bool > DisableSchedRegPressure("disable-sched-reg-pressure", cl::Hidden, cl::init(false), cl::desc("Disable regpressure priority in sched=list-ilp"))
static cl::opt< unsigned > AvgIPC("sched-avg-ipc", cl::Hidden, cl::init(1), cl::desc("Average inst/cycle whan no target itinerary exists."))
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
Describe properties that are true of each instruction in the target description file.
MachineInstrBuilder MachineInstrBuilder &DefMI const MCInstrDesc & Desc
static void GetCostForDef(const ScheduleDAGSDNodes::RegDefIter &RegDefPos, const TargetLowering *TLI, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, unsigned &RegClass, unsigned &Cost, const MachineFunction &MF)
GetCostForDef - Looks up the register class and cost for a given definition.
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getID() const
Return the register class ID number.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
static bool isClobberKind(unsigned Flag)
static bool canClobberReachingPhysRegUse(const SUnit *DepSU, const SUnit *SU, ScheduleDAGRRList *scheduleDAG, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
canClobberReachingPhysRegUse - True if SU would clobber one of it's successor's explicit physregs who...
const SDValue & getOperand(unsigned Num) const
void removePred(const SDep &D)
removePred - This removes the specified edge as a pred of the current node if it exists.
static cl::opt< bool > DisableSchedVRegCycle("disable-sched-vrcycle", cl::Hidden, cl::init(false), cl::desc("Disable virtual register cycle interference checks"))
void setNodeId(int Id)
Set unique node id.
static unsigned CalcNodeSethiUllmanNumber(const SUnit *SU, std::vector< unsigned > &SUNumbers)
CalcNodeSethiUllmanNumber - Compute Sethi Ullman number.
bool isAssignedRegDep() const
isAssignedRegDep - Test if this is a Data dependence that is associated with a register.
static bool hasVRegCycleUse(const SUnit *SU)
const TargetRegisterClass * CopyDstRC
EntryToken - This is the marker used to indicate the start of a region.
bool isArtificial() const
isArtificial - Test if this is an Order dependence that is marked as "artificial", meaning it isn't necessary for correctness.
virtual const TargetRegisterClass * getRepRegClassFor(MVT VT) const
Return the 'representative' register class for the specified value type.
SmallVector< SDep, 4 > Preds
static SDNode * FindCallSeqStart(SDNode *N, unsigned &NestLevel, unsigned &MaxNest, const TargetInstrInfo *TII)
FindCallSeqStart - Starting from the (lowered) CALLSEQ_END node, locate the corresponding (lowered) C...
void dumpAll(const ScheduleDAG *G) const
SchedulingPriorityQueue - This interface is used to plug different priorities computation algorithms ...
unsigned getHeight() const
getHeight - Return the height of this node, which is the length of the maximum path down to any node ...
unsigned getCallFrameDestroyOpcode() const
static int checkSpecialNodes(const SUnit *left, const SUnit *right)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
struct fuzzer::@269 Flags
const HexagonInstrInfo * TII
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
const TargetRegisterClass * getRegClass(unsigned i) const
Returns the register class associated with the enumeration value.
const TargetLowering * TLI
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
Regular data dependence (aka true-dependence).
CopyToReg - This node has three operands: a chain, a register number to set to this value...
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
Reg
All possible values of the reg field in the ModR/M byte.
static bool isRegDefEarlyClobberKind(unsigned Flag)
static void CheckForLiveRegDef(SUnit *SU, unsigned Reg, SUnit **LiveRegDefs, SmallSet< unsigned, 4 > &RegAdded, SmallVectorImpl< unsigned > &LRegs, const TargetRegisterInfo *TRI)
CheckForLiveRegDef - Return true and update live register vector if the specified register def of the...
static const uint32_t * getNodeRegMask(const SDNode *N)
getNodeRegMask - Returns the register mask attached to an SDNode, if any.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
INLINEASM - Represents an inline asm block.
LLVM_NODISCARD bool empty() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
unsigned getIROrder() const
Return the node ordering.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
ScheduleDAGSDNodes * createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createHybridListDAGScheduler - This creates a bottom up register pressure aware list scheduler that m...
unsigned getCallFrameSetupOpcode() const
These methods return the opcode of the frame setup/destroy instructions if they exist (-1 otherwise)...
const MCPhysReg * getImplicitDefs() const
Return a list of registers that are potentially written by any instance of this machine instruction...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
static bool isRegDefKind(unsigned Flag)
ScheduleDAGSDNodes * createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up list scheduler that schedules nodes in source c...
size_t size() const
size - Get the array size.
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
static RegisterScheduler burrListDAGScheduler("list-burr","Bottom-up register reduction list scheduling", createBURRListDAGScheduler)
TargetInstrInfo - Interface to description of machine instruction set.
This corresponds to the llvm.lifetime.
static void CheckForLiveRegDefMasked(SUnit *SU, const uint32_t *RegMask, ArrayRef< SUnit * > LiveRegDefs, SmallSet< unsigned, 4 > &RegAdded, SmallVectorImpl< unsigned > &LRegs)
CheckForLiveRegDefMasked - Check for any live physregs that are clobbered by RegMask, and add them to LRegs.
SDep - Scheduling dependency.
SDNode * getNode() const
get the SDNode which holds the desired result
void setHeightToAtLeast(unsigned NewHeight)
setDepthToAtLeast - If NewDepth is greater than this node's depth value, set it to be the new height ...
initializer< Ty > init(const Ty &Val)
static RegisterScheduler hybridListDAGScheduler("list-hybrid","Bottom-up register pressure aware list scheduling ""which tries to balance latency and register pressure", createHybridListDAGScheduler)
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
bool regsOverlap(unsigned regA, unsigned regB) const
Returns true if the two registers are equal or alias each other.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
static cl::opt< bool > DisableSchedStalls("disable-sched-stalls", cl::Hidden, cl::init(true), cl::desc("Disable no-stall priority in sched=list-ilp"))
MVT - Machine Value Type.
HazardRecognizer - This determines whether or not an instruction can be issued this cycle...
SDNode * getNode() const
getNode - Return the representative SDNode for this SUnit.
ScheduleDAGSDNodes * createBURRListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up register usage reduction list scheduler...
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
static bool isOperandOf(const SUnit *SU, SDNode *N)
static RegisterScheduler ILPListDAGScheduler("list-ilp","Bottom-up register pressure aware list scheduling ""which tries to balance ILP and register pressure", createILPListDAGScheduler)
static unsigned calcMaxScratches(const SUnit *SU)
calcMaxScratches - Returns an cost estimate of the worse case requirement for scratch registers...
static bool IsChainDependent(SDNode *Outer, SDNode *Inner, unsigned NestLevel, const TargetInstrInfo *TII)
IsChainDependent - Test if Outer is reachable from Inner through chain dependencies.
static bool BUHasStall(SUnit *SU, int Height, RegReductionPQBase *SPQ)
const MCPhysReg * ImplicitDefs
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
static cl::opt< bool > DisableSchedCriticalPath("disable-sched-critical-path", cl::Hidden, cl::init(false), cl::desc("Disable critical path priority in sched=list-ilp"))
MCRegAliasIterator enumerates all registers aliasing Reg.
unsigned getLatency() const
getLatency - Return the latency value for this edge, which roughly means the minimum number of cycles...
for(unsigned i=0, e=MI->getNumOperands();i!=e;++i)
static bool canClobberPhysRegDefs(const SUnit *SuccSU, const SUnit *SU, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
canClobberPhysRegDefs - True if SU would clobber one of SuccSU's physical register defs...
static void resetVRegCycle(SUnit *SU)
static cl::opt< bool > DisableSchedLiveUses("disable-sched-live-uses", cl::Hidden, cl::init(true), cl::desc("Disable live use priority in sched=list-ilp"))
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
static bool hasOnlyLiveInOpers(const SUnit *SU)
hasOnlyLiveInOpers - Return true if SU has only value predecessors that are CopyFromReg from a virtua...
static unsigned closestSucc(const SUnit *SU)
closestSucc - Returns the scheduled cycle of the successor which is closest to the current cycle...
Sched::Preference SchedulingPref
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
RegDefIter - In place iteration over the values defined by an SUnit.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specific constraint if it is set.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
static bool BURRSort(SUnit *left, SUnit *right, RegReductionPQBase *SPQ)
TokenFactor - This node takes multiple tokens as input and produces a single token result...
ScheduleDAGSDNodes * createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createILPListDAGScheduler - This creates a bottom up register pressure aware list scheduler that trie...
static cl::opt< bool > DisableSchedHeight("disable-sched-height", cl::Hidden, cl::init(false), cl::desc("Disable scheduled-height priority in sched=list-ilp"))
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
static void initVRegCycle(SUnit *SU)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
const TargetRegisterClass * CopySrcRC
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static bool clobbersPhysReg(const uint32_t *RegMask, unsigned PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
unsigned getDepth() const
getDepth - Return the depth of this node, which is the length of the maximum path up to any node whic...
iterator_range< value_op_iterator > op_values() const
void setLatency(unsigned Lat)
setLatency - Set the latency for this edge.
TargetSubtargetInfo - Generic base class for all target subtargets.
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
static bool hasOnlyLiveOutUses(const SUnit *SU)
hasOnlyLiveOutUses - Return true if SU has only value successors that are CopyToReg to a virtual regi...
static cl::opt< bool > DisableSchedPhysRegJoin("disable-sched-physreg-join", cl::Hidden, cl::init(false), cl::desc("Disable physreg def-use affinity"))
static cl::opt< int > MaxReorderWindow("max-sched-reorder", cl::Hidden, cl::init(6), cl::desc("Number of instructions to allow ahead of the critical path ""in sched=list-ilp"))
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Sequence
A sequence of states that a pointer may go through in which an objc_retain and objc_release are actua...
unsigned short NumRegDefsLeft
static cl::opt< bool > Disable2AddrHack("disable-2addr-hack", cl::Hidden, cl::init(true), cl::desc("Disable scheduler's two-address hack"))
static MVT getPhysicalRegisterVT(SDNode *N, unsigned Reg, const TargetInstrInfo *TII)
getPhysicalRegisterVT - Returns the ValueType of the physical register definition of the specified no...
static cl::opt< bool > DisableSchedCycles("disable-sched-cycles", cl::Hidden, cl::init(false), cl::desc("Disable cycle-level precision during preRA scheduling"))
int getNodeId() const
Return the unique node id.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
static RegisterScheduler sourceListDAGScheduler("source","Similar to list-burr but schedules in source ""order when possible", createSourceListDAGScheduler)
virtual unsigned getRegPressureLimit(const TargetRegisterClass *RC, MachineFunction &MF) const
Return the register pressure "high water mark" for the specific register class.
virtual ScheduleHazardRecognizer * CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, const ScheduleDAG *DAG) const
Allocate and return a hazard recognizer to use for this target when scheduling the machine instructio...
void setHeightDirty()
setHeightDirty - Set a flag in this node to indicate that its stored Height value will require recomp...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool addPred(const SDep &D, bool Required=true)
addPred - This adds the specified edge as a pred of the current node if not already.
virtual const TargetInstrInfo * getInstrInfo() const
SmallVector< SDep, 4 > Succs
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Arbitrary strong DAG edge (no real dependence).
virtual uint8_t getRepRegClassCostFor(MVT VT) const
Return the cost of the 'representative' register class for the specified value type.
static bool isLiveOut(const MachineBasicBlock &MBB, unsigned Reg)
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
const TargetRegisterClass * getRegClass(const MCInstrDesc &TID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum...
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
unsigned getReg() const
getReg - Return the register associated with this edge.
ScheduleDAGTopologicalSort is a class that computes a topological ordering for SUnits and provides me...
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
const TargetRegisterClass *const * regclass_iterator
void dump(const ScheduleDAG *G) const
SUnit - Scheduling unit.
SUnit - Scheduling unit. This is a node in the scheduling DAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
This file describes how to lower LLVM code to machine code.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.