111 #include <functional>
118 using namespace llvm;
120 #define DEBUG_TYPE "pipeliner"
122 STATISTIC(NumTrytoPipeline,
"Number of loops that we attempt to pipeline");
123 STATISTIC(NumPipelined,
"Number of loops software pipelined");
128 cl::desc(
"Enable Software Pipelining"));
137 cl::desc(
"Size limit for the the MII."),
143 cl::desc(
"Maximum stages allowed in the generated scheduled."),
150 cl::desc(
"Prune dependences between unrelated Phi nodes."),
157 cl::desc(
"Prune loop carried order dependences."),
172 class SwingSchedulerDAG;
223 MachinePipeliner &
Pass;
239 NodeInfo() : ASAP(0), ALAP(0) {}
242 std::vector<NodeInfo> ScheduleInfo;
244 enum OrderKind { BottomUp = 0, TopDown = 1 };
261 std::vector<std::unique_ptr<ScheduleDAGMutation>> Mutations;
265 std::vector<SUnit> &SUnits;
271 static unsigned MaxPaths;
274 Circuits(std::vector<SUnit> &SUs)
275 : SUnits(SUs),
Stack(), Blocked(SUs.size()),
B(SUs.size()),
284 void createAdjacencyStructure(SwingSchedulerDAG *DAG);
285 bool circuit(
int V,
int S, NodeSetType &NodeSets,
bool HasBackedge =
false);
293 Scheduled(
false),
Loop(L), LIS(lis), RegClassInfo(rci),
294 Topo(SUnits, &ExitSU) {
295 P.MF->getSubtarget().getSMSMutations(Mutations);
298 void schedule()
override;
299 void finishBlock()
override;
302 bool hasNewSchedule() {
return Scheduled; }
305 int getASAP(
SUnit *Node) {
return ScheduleInfo[Node->
NodeNum].ASAP; }
308 int getALAP(
SUnit *Node) {
return ScheduleInfo[Node->
NodeNum].ALAP; }
312 int getMOV(
SUnit *Node) {
return getALAP(Node) - getASAP(Node); }
330 static bool isOrder(
SUnit *Source,
const SDep &Dep) {
337 bool isLoopCarriedOrder(
SUnit *Source,
const SDep &Dep,
bool isSucc =
true);
364 void setMII(
unsigned mii) { MII = mii; }
367 bool UpdateDAG =
false);
371 unsigned getInstrBaseReg(
SUnit *SU) {
373 InstrChanges.
find(SU);
374 if (It != InstrChanges.
end())
375 return It->second.first;
379 void addMutation(std::unique_ptr<ScheduleDAGMutation>
Mutation) {
380 Mutations.push_back(std::move(Mutation));
385 void updatePhiDependences();
386 void changeDependences();
387 unsigned calculateResMII();
388 unsigned calculateRecMII(NodeSetType &RecNodeSets);
389 void findCircuits(NodeSetType &NodeSets);
390 void fuseRecs(NodeSetType &NodeSets);
391 void removeDuplicateNodes(NodeSetType &NodeSets);
392 void computeNodeFunctions(NodeSetType &NodeSets);
393 void registerPressureFilter(NodeSetType &NodeSets);
394 void colocateNodeSets(NodeSetType &NodeSets);
395 void checkNodeSets(NodeSetType &NodeSets);
396 void groupRemainingNodes(NodeSetType &NodeSets);
399 void computeNodeOrder(NodeSetType &NodeSets);
400 bool schedulePipeline(SMSchedule &Schedule);
401 void generatePipelinedLoop(SMSchedule &Schedule);
402 void generateProlog(SMSchedule &Schedule,
unsigned LastStage,
404 MBBVectorTy &PrologBBs);
405 void generateEpilog(SMSchedule &Schedule,
unsigned LastStage,
407 MBBVectorTy &EpilogBBs, MBBVectorTy &PrologBBs);
410 SMSchedule &Schedule, ValueMapTy *VRMap,
411 InstrMapTy &InstrMap,
unsigned LastStageNum,
412 unsigned CurStageNum,
bool IsLast);
415 SMSchedule &Schedule, ValueMapTy *VRMap,
416 InstrMapTy &InstrMap,
unsigned LastStageNum,
417 unsigned CurStageNum,
bool IsLast);
419 MBBVectorTy &EpilogBBs);
421 SMSchedule &Schedule);
423 MBBVectorTy &EpilogBBs, SMSchedule &Schedule,
429 unsigned InstStageNum);
431 unsigned InstStageNum,
432 SMSchedule &Schedule);
433 void updateInstruction(
MachineInstr *NewMI,
bool LastDef,
434 unsigned CurStageNum,
unsigned InstStageNum,
435 SMSchedule &Schedule, ValueMapTy *VRMap);
437 unsigned getPrevMapVal(
unsigned StageNum,
unsigned PhiStage,
unsigned LoopVal,
438 unsigned LoopStage, ValueMapTy *VRMap,
441 SMSchedule &Schedule, ValueMapTy *VRMap,
442 InstrMapTy &InstrMap);
444 InstrMapTy &InstrMap,
unsigned CurStageNum,
446 unsigned OldReg,
unsigned NewReg,
447 unsigned PrevReg = 0);
449 unsigned &OffsetPos,
unsigned &NewBase,
451 void postprocessDAG();
462 unsigned Colocate = 0;
463 SUnit *ExceedPressure =
nullptr;
470 NodeSet(iterator S, iterator
E) : Nodes(S, E), HasRecurrence(
true) {}
472 bool insert(
SUnit *SU) {
return Nodes.insert(SU); }
474 void insert(iterator S, iterator
E) { Nodes.insert(S, E); }
476 template <
typename UnaryPredicate>
bool remove_if(UnaryPredicate P) {
477 return Nodes.remove_if(P);
480 unsigned count(
SUnit *SU)
const {
return Nodes.count(SU); }
482 bool hasRecurrence() {
return HasRecurrence; };
484 unsigned size()
const {
return Nodes.size(); }
486 bool empty()
const {
return Nodes.empty(); }
488 SUnit *getNode(
unsigned i)
const {
return Nodes[
i]; };
490 void setRecMII(
unsigned mii) { RecMII = mii; };
492 void setColocate(
unsigned c) { Colocate = c; };
494 void setExceedPressure(
SUnit *SU) { ExceedPressure = SU; }
496 bool isExceedSU(
SUnit *SU) {
return ExceedPressure == SU; }
498 int compareRecMII(
NodeSet &RHS) {
return RecMII - RHS.RecMII; }
500 int getRecMII() {
return RecMII; }
503 void computeNodeSetInfo(SwingSchedulerDAG *SSD) {
504 for (
SUnit *SU : *
this) {
505 MaxMOV = std::max(MaxMOV, SSD->getMOV(SU));
506 MaxDepth = std::max(MaxDepth, SSD->getDepth(SU));
513 HasRecurrence =
false;
517 ExceedPressure =
nullptr;
527 if (RecMII == RHS.RecMII) {
528 if (Colocate != 0 && RHS.Colocate != 0 && Colocate != RHS.Colocate)
529 return Colocate < RHS.Colocate;
530 if (MaxMOV == RHS.MaxMOV)
532 return MaxMOV < RHS.MaxMOV;
534 return RecMII > RHS.RecMII;
538 return RecMII == RHS.RecMII && MaxMOV == RHS.MaxMOV &&
545 iterator
end() {
return Nodes.
end(); }
548 os <<
"Num nodes " << size() <<
" rec " << RecMII <<
" mov " << MaxMOV
549 <<
" depth " <<
MaxDepth <<
" col " << Colocate <<
"\n";
550 for (
const auto &
I : Nodes)
551 os <<
" SU(" <<
I->NodeNum <<
") " << *(
I->getInstr());
555 void dump()
const { print(
dbgs()); }
572 std::map<SUnit *, int> InstrToCycle;
578 std::map<unsigned, std::pair<unsigned, bool>> RegToStageDiff;
588 int InitiationInterval;
600 :
ST(mf->getSubtarget()),
MRI(mf->getRegInfo()),
601 Resources(
ST.getInstrInfo()->CreateTargetScheduleState(
ST)) {
604 InitiationInterval = 0;
608 ScheduledInstrs.clear();
609 InstrToCycle.clear();
610 RegToStageDiff.clear();
615 ScheduledInstrs.clear();
616 InstrToCycle.clear();
617 RegToStageDiff.clear();
620 InitiationInterval = 0;
624 void setInitiationInterval(
int ii) { InitiationInterval = ii; }
628 int getFirstCycle()
const {
return FirstCycle; }
631 int getFinalCycle()
const {
return FirstCycle + InitiationInterval - 1; }
635 int earliestCycleInChain(
const SDep &Dep);
639 int latestCycleInChain(
const SDep &Dep);
641 void computeStart(
SUnit *SU,
int *MaxEarlyStart,
int *MinLateStart,
642 int *MinEnd,
int *MaxStart,
int II, SwingSchedulerDAG *DAG);
643 bool insert(
SUnit *SU,
int StartCycle,
int EndCycle,
int II);
648 const_sched_iterator;
651 bool isScheduledAtStage(
SUnit *SU,
unsigned StageNum) {
652 return (stageScheduled(SU) == (
int)StageNum);
657 int stageScheduled(
SUnit *SU)
const {
658 std::map<SUnit *, int>::const_iterator it = InstrToCycle.
find(SU);
659 if (it == InstrToCycle.end())
661 return (it->second - FirstCycle) / InitiationInterval;
666 unsigned cycleScheduled(
SUnit *SU)
const {
667 std::map<SUnit *, int>::const_iterator it = InstrToCycle.find(SU);
668 assert(it != InstrToCycle.end() &&
"Instruction hasn't been scheduled.");
669 return (it->second - FirstCycle) % InitiationInterval;
673 unsigned getMaxStageCount() {
674 return (LastCycle - FirstCycle) / InitiationInterval;
679 unsigned getStagesForReg(
int Reg,
unsigned CurStage) {
680 std::pair<unsigned, bool> Stages = RegToStageDiff[
Reg];
681 if (CurStage > getMaxStageCount() && Stages.first == 0 && Stages.second)
692 unsigned getStagesForPhi(
int Reg) {
693 std::pair<unsigned, bool> Stages = RegToStageDiff[
Reg];
696 return Stages.first - 1;
700 std::deque<SUnit *> &getInstructions(
int cycle) {
701 return ScheduledInstrs[cycle];
704 bool isValidSchedule(SwingSchedulerDAG *SSD);
705 void finalizeSchedule(SwingSchedulerDAG *SSD);
706 bool orderDependence(SwingSchedulerDAG *SSD,
SUnit *SU,
707 std::deque<SUnit *> &Insts);
708 bool isLoopCarried(SwingSchedulerDAG *SSD,
MachineInstr &Phi);
709 bool isLoopCarriedDefOfUse(SwingSchedulerDAG *SSD,
MachineInstr *Inst,
717 unsigned SwingSchedulerDAG::Circuits::MaxPaths = 5;
720 int MachinePipeliner::NumTries = 0;
724 "Modulo Software Pipelining",
false,
false)
734 if (skipFunction(*mf.getFunction()))
740 if (mf.getFunction()->getAttributes().hasAttribute(
746 MLI = &getAnalysis<MachineLoopInfo>();
747 MDT = &getAnalysis<MachineDominatorTree>();
748 TII = MF->getSubtarget().getInstrInfo();
749 RegClassInfo.runOnMachineFunction(*MF);
761 bool MachinePipeliner::scheduleLoop(
MachineLoop &L) {
762 bool Changed =
false;
763 for (
auto &InnerLoop : L)
764 Changed |= scheduleLoop(*InnerLoop);
776 if (!canPipelineLoop(L))
781 Changed = swingModuloScheduler(L);
789 bool MachinePipeliner::canPipelineLoop(
MachineLoop &L) {
801 LI.LoopInductionVar =
nullptr;
802 LI.LoopCompare =
nullptr;
816 for (
unsigned i = 1;
i != BBI->getNumOperands();
i += 2)
817 if (BBI->getOperand(
i).getSubReg() != 0)
827 bool MachinePipeliner::swingModuloScheduler(
MachineLoop &L) {
828 assert(L.
getBlocks().size() == 1 &&
"SMS works on single blocks only.");
830 SwingSchedulerDAG SMS(*
this, L, getAnalysis<LiveIntervals>(), RegClassInfo);
839 unsigned size = MBB->
size();
850 return SMS.hasNewSchedule();
855 void SwingSchedulerDAG::schedule() {
858 addLoopCarriedDependences(AA);
859 updatePhiDependences();
860 Topo.InitDAGTopologicalSorting();
864 for (
unsigned su = 0, e = SUnits.size(); su != e; ++su)
865 SUnits[su].dumpAll(
this);
868 NodeSetType NodeSets;
869 findCircuits(NodeSets);
872 unsigned ResMII = calculateResMII();
873 unsigned RecMII = calculateRecMII(NodeSets);
881 MII = std::max(ResMII, RecMII);
882 DEBUG(
dbgs() <<
"MII = " << MII <<
" (rec=" << RecMII <<
", res=" << ResMII
893 computeNodeFunctions(NodeSets);
895 registerPressureFilter(NodeSets);
897 colocateNodeSets(NodeSets);
899 checkNodeSets(NodeSets);
902 for (
auto &
I : NodeSets) {
903 dbgs() <<
" Rec NodeSet ";
908 std::sort(NodeSets.begin(), NodeSets.end(), std::greater<NodeSet>());
910 groupRemainingNodes(NodeSets);
912 removeDuplicateNodes(NodeSets);
915 for (
auto &
I : NodeSets) {
916 dbgs() <<
" NodeSet ";
921 computeNodeOrder(NodeSets);
923 SMSchedule Schedule(
Pass.MF);
924 Scheduled = schedulePipeline(Schedule);
929 unsigned numStages = Schedule.getMaxStageCount();
938 generatePipelinedLoop(Schedule);
943 void SwingSchedulerDAG::finishBlock() {
945 MF.DeleteMachineInstr(
I);
955 unsigned &InitVal,
unsigned &LoopVal) {
966 assert(InitVal != 0 && LoopVal != 0 &&
"Unexpected Phi structure.");
990 while (!Worklist.
empty()) {
995 if (Visited.
count(SuccSU))
1033 void SwingSchedulerDAG::addLoopCarriedDependences(
AliasAnalysis *AA) {
1035 for (
auto &SU : SUnits) {
1038 PendingLoads.
clear();
1042 for (
auto V : Objs) {
1049 for (
auto V : Objs) {
1051 PendingLoads.
find(V);
1052 if (I == PendingLoads.
end())
1054 for (
auto Load : I->second) {
1061 unsigned BaseReg1, BaseReg2;
1062 int64_t Offset1, Offset2;
1068 if (BaseReg1 == BaseReg2 && (
int)Offset1 < (
int)Offset2) {
1070 "What happened to the chain edge?");
1112 void SwingSchedulerDAG::updatePhiDependences() {
1117 for (
SUnit &I : SUnits) {
1120 unsigned HasPhiUse = 0;
1121 unsigned HasPhiDef = 0;
1126 MOI != MOE; ++MOI) {
1129 unsigned Reg = MOI->getReg();
1133 UI =
MRI.use_instr_begin(Reg),
1134 UE =
MRI.use_instr_end();
1137 SUnit *SU = getSUnit(UseMI);
1138 if (SU !=
nullptr && UseMI->
isPHI()) {
1146 if (SU->
NodeNum < I.NodeNum && !I.isPred(SU))
1151 }
else if (MOI->isUse()) {
1154 if (DefMI ==
nullptr)
1156 SUnit *SU = getSUnit(DefMI);
1157 if (SU !=
nullptr && DefMI->
isPHI()) {
1167 if (SU->
NodeNum < I.NodeNum && !I.isPred(SU))
1176 for (
auto &PI : I.Preds) {
1179 if (I.getInstr()->isPHI()) {
1188 for (
int i = 0, e = RemoveDeps.
size();
i != e; ++
i)
1189 I.removePred(RemoveDeps[
i]);
1195 void SwingSchedulerDAG::changeDependences() {
1199 for (
SUnit &I : SUnits) {
1200 unsigned BasePos = 0, OffsetPos = 0, NewBase = 0;
1201 int64_t NewOffset = 0;
1202 if (!canUseLastOffsetValue(I.getInstr(), BasePos, OffsetPos, NewBase,
1207 unsigned OrigBase = I.getInstr()->getOperand(BasePos).getReg();
1211 SUnit *DefSU = getSUnit(DefMI);
1218 SUnit *LastSU = getSUnit(LastMI);
1222 if (Topo.IsReachable(&I, LastSU))
1229 if (P->getSUnit() == DefSU)
1231 for (
int i = 0, e = Deps.
size();
i != e;
i++) {
1232 Topo.RemovePred(&I, Deps[
i].getSUnit());
1233 I.removePred(Deps[
i]);
1237 for (
auto &P : LastSU->
Preds)
1238 if (P.getSUnit() == &I && P.getKind() ==
SDep::Order)
1240 for (
int i = 0, e = Deps.
size();
i != e;
i++) {
1241 Topo.RemovePred(LastSU, Deps[
i].getSUnit());
1252 InstrChanges[&
I] = std::make_pair(NewBase, NewOffset);
1260 struct FuncUnitSorter {
1267 unsigned minFuncUnits(
const MachineInstr *Inst,
unsigned &
F)
const {
1269 unsigned min = UINT_MAX;
1270 for (
const InstrStage *IS = InstrItins->beginStage(schedClass),
1271 *
IE = InstrItins->endStage(schedClass);
1273 unsigned funcUnits = IS->getUnits();
1275 if (numAlternatives < min) {
1276 min = numAlternatives;
1290 for (
const InstrStage *IS = InstrItins->beginStage(SchedClass),
1291 *
IE = InstrItins->endStage(SchedClass);
1293 unsigned FuncUnits = IS->getUnits();
1295 Resources[FuncUnits]++;
1302 unsigned F1 = 0, F2 = 0;
1303 unsigned MFUs1 = minFuncUnits(IS1, F1);
1304 unsigned MFUs2 = minFuncUnits(IS2, F2);
1305 if (MFUs1 == 1 && MFUs2 == 1)
1306 return Resources.lookup(F1) < Resources.lookup(F2);
1307 return MFUs1 > MFUs2;
1319 unsigned SwingSchedulerDAG::calculateResMII() {
1326 FuncUnitSorter FUS =
1327 FuncUnitSorter(MF.getSubtarget().getInstrItineraryData());
1331 FUS.calcCriticalResources(*I);
1338 FuncUnitOrder.push(&*I);
1340 while (!FuncUnitOrder.empty()) {
1342 FuncUnitOrder.pop();
1347 unsigned NumCycles = getSUnit(MI)->Latency;
1348 unsigned ReservedCycles = 0;
1351 for (
unsigned C = 0;
C < NumCycles; ++
C)
1353 if ((*RI++)->canReserveResources(*MI)) {
1359 for (
unsigned C = 0;
C < ReservedCycles; ++
C) {
1361 (*RI)->reserveResources(*MI);
1364 for (
unsigned C = ReservedCycles;
C < NumCycles; ++
C) {
1372 int Resmii = Resources.
size();
1388 unsigned SwingSchedulerDAG::calculateRecMII(NodeSetType &NodeSets) {
1389 unsigned RecMII = 0;
1391 for (
NodeSet &Nodes : NodeSets) {
1392 if (Nodes.size() == 0)
1395 unsigned Delay = Nodes.size() - 1;
1396 unsigned Distance = 1;
1399 unsigned CurMII = (Delay + Distance - 1) / Distance;
1400 Nodes.setRecMII(CurMII);
1401 if (CurMII > RecMII)
1412 for (
unsigned i = 0, e = SUnits.size();
i != e; ++
i) {
1418 DepsAdded.
push_back(std::make_pair(SU, *IP));
1421 for (
SmallVector<std::pair<SUnit *, SDep>, 8>::iterator I = DepsAdded.
begin(),
1422 E = DepsAdded.
end();
1425 SUnit *SU = I->first;
1426 SDep &
D = I->second;
1428 unsigned Reg = D.
getReg();
1438 void SwingSchedulerDAG::Circuits::createAdjacencyStructure(
1439 SwingSchedulerDAG *DAG) {
1441 for (
int i = 0, e = SUnits.size();
i != e; ++
i) {
1444 for (
auto &SI : SUnits[
i].Succs) {
1447 if (
SI.getSUnit()->isBoundaryNode() ||
1448 (
SI.getKind() ==
SDep::Anti && !
SI.getSUnit()->getInstr()->isPHI()))
1450 int N =
SI.getSUnit()->NodeNum;
1451 if (!Added.test(N)) {
1452 AdjK[
i].push_back(N);
1458 for (
auto &PI : SUnits[
i].Preds) {
1459 if (!SUnits[
i].getInstr()->mayStore() ||
1460 !DAG->isLoopCarriedOrder(&SUnits[
i], PI,
false))
1462 if (PI.getKind() ==
SDep::Order && PI.getSUnit()->getInstr()->mayLoad()) {
1463 int N = PI.getSUnit()->NodeNum;
1464 if (!Added.test(N)) {
1465 AdjK[
i].push_back(N);
1475 bool SwingSchedulerDAG::Circuits::circuit(
int V,
int S, NodeSetType &NodeSets,
1477 SUnit *SV = &SUnits[V];
1482 for (
auto W : AdjK[V]) {
1483 if (NumPaths > MaxPaths)
1493 }
else if (!Blocked.test(W)) {
1494 if (circuit(W, S, NodeSets, W < V ?
true : HasBackedge))
1502 for (
auto W : AdjK[V]) {
1514 void SwingSchedulerDAG::Circuits::unblock(
int U) {
1517 while (!BU.
empty()) {
1519 assert(SI != BU.
end() &&
"Invalid B set.");
1522 if (Blocked.test(W->NodeNum))
1523 unblock(W->NodeNum);
1529 void SwingSchedulerDAG::findCircuits(NodeSetType &NodeSets) {
1534 Circuits Cir(SUnits);
1536 Cir.createAdjacencyStructure(
this);
1537 for (
int i = 0, e = SUnits.size(); i != e; ++
i) {
1539 Cir.circuit(i, i, NodeSets);
1561 void SwingSchedulerDAG::computeNodeFunctions(NodeSetType &NodeSets) {
1563 ScheduleInfo.resize(SUnits.size());
1582 EP = SU->
Preds.end();
1587 asap = std::max(asap, (
int)(getASAP(pred) +
getLatency(SU, *IP) -
1588 getDistance(pred, SU, *IP) * MII));
1590 maxASAP = std::max(maxASAP, asap);
1591 ScheduleInfo[*
I].ASAP = asap;
1605 SUnit *succ = IS->getSUnit();
1607 getDistance(SU, succ, *IS) * MII));
1610 ScheduleInfo[*
I].ALAP = alap;
1615 I.computeNodeSetInfo(
this);
1618 for (
unsigned i = 0; i < SUnits.size(); i++) {
1619 dbgs() <<
"\tNode " << i <<
":\n";
1620 dbgs() <<
"\t ASAP = " << getASAP(&SUnits[i]) <<
"\n";
1621 dbgs() <<
"\t ALAP = " << getALAP(&SUnits[i]) <<
"\n";
1622 dbgs() <<
"\t MOV = " << getMOV(&SUnits[i]) <<
"\n";
1623 dbgs() <<
"\t D = " << getDepth(&SUnits[i]) <<
"\n";
1624 dbgs() <<
"\t H = " << getHeight(&SUnits[i]) <<
"\n";
1640 if (S && S->count(PI->getSUnit()) == 0)
1644 if (NodeOrder.
count(PI->getSUnit()) == 0)
1645 Preds.
insert(PI->getSUnit());
1649 ES = (*I)->Succs.end();
1653 if (S && S->count(IS->getSUnit()) == 0)
1655 if (NodeOrder.
count(IS->getSUnit()) == 0)
1656 Preds.
insert(IS->getSUnit());
1659 return Preds.
size() > 0;
1673 if (S && S->count(SI->getSUnit()) == 0)
1677 if (NodeOrder.
count(SI->getSUnit()) == 0)
1678 Succs.
insert(SI->getSUnit());
1681 PE = (*I)->Preds.end();
1685 if (S && S->count(PI->getSUnit()) == 0)
1687 if (NodeOrder.
count(PI->getSUnit()) == 0)
1688 Succs.
insert(PI->getSUnit());
1691 return Succs.
size() > 0;
1702 if (Exclude.
count(Cur) != 0)
1704 if (DestNodes.
count(Cur) != 0)
1706 if (!Visited.
insert(Cur).second)
1707 return Path.
count(Cur) != 0;
1708 bool FoundPath =
false;
1709 for (
auto &SI : Cur->
Succs)
1710 FoundPath |=
computePath(SI.getSUnit(), Path, DestNodes, Exclude, Visited);
1711 for (
auto &PI : Cur->
Preds)
1714 computePath(PI.getSUnit(), Path, DestNodes, Exclude, Visited);
1721 template <
class S1Ty,
class S2Ty>
static bool isSubset(S1Ty &Set1, S2Ty &Set2) {
1722 for (
typename S1Ty::iterator I = Set1.
begin(), E = Set1.end(); I !=
E; ++
I)
1723 if (Set2.count(*I) == 0)
1737 for (
SUnit *SU : NS) {
1742 if (MO.isReg() && MO.isUse()) {
1743 unsigned Reg = MO.getReg();
1751 for (
SUnit *SU : NS)
1753 if (MO.isReg() && MO.isDef() && !MO.isDead()) {
1754 unsigned Reg = MO.getReg();
1756 if (!Uses.
count(Reg))
1761 if (!Uses.
count(*Units))
1771 void SwingSchedulerDAG::registerPressureFilter(NodeSetType &NodeSets) {
1772 for (
auto &NS : NodeSets) {
1778 RecRPTracker.init(&MF, &RegClassInfo, &LIS, BB, BB->end(),
false,
true);
1780 RecRPTracker.closeBottom();
1782 std::vector<SUnit *> SUnits(NS.begin(), NS.end());
1783 std::sort(SUnits.begin(), SUnits.end(), [](
const SUnit *
A,
const SUnit *
B) {
1787 for (
auto &SU : SUnits) {
1793 RecRPTracker.setPos(std::next(CurInstI));
1797 RecRPTracker.getMaxUpwardPressureDelta(SU->
getInstr(),
nullptr, RPDelta,
1804 NS.setExceedPressure(SU);
1807 RecRPTracker.recede();
1814 void SwingSchedulerDAG::colocateNodeSets(NodeSetType &NodeSets) {
1815 unsigned Colocate = 0;
1816 for (
int i = 0, e = NodeSets.size(); i < e; ++
i) {
1819 if (N1.empty() || !
succ_L(N1, S1))
1821 for (
int j = i + 1; j < e; ++j) {
1823 if (N1.compareRecMII(N2) != 0)
1826 if (N2.empty() || !
succ_L(N2, S2))
1829 N1.setColocate(++Colocate);
1830 N2.setColocate(Colocate);
1842 void SwingSchedulerDAG::checkNodeSets(NodeSetType &NodeSets) {
1847 for (
auto &NS : NodeSets)
1852 for (
auto &SU : SUnits)
1855 DEBUG(
dbgs() <<
"Clear recurrence node-sets\n");
1862 void SwingSchedulerDAG::groupRemainingNodes(NodeSetType &NodeSets) {
1872 for (
SUnit *NI : N) {
1876 if (Path.
size() > 0)
1881 if (
succ_L(NodesAdded, N)) {
1883 for (
SUnit *NI : N) {
1887 if (Path.
size() > 0)
1897 if (
succ_L(NodesAdded, N))
1899 addConnectedNodes(I, NewSet, NodesAdded);
1900 if (NewSet.size() > 0)
1901 NodeSets.push_back(NewSet);
1906 if (
pred_L(NodesAdded, N))
1908 addConnectedNodes(I, NewSet, NodesAdded);
1909 if (NewSet.size() > 0)
1910 NodeSets.push_back(NewSet);
1914 for (
unsigned i = 0; i < SUnits.size(); ++
i) {
1916 if (NodesAdded.
count(SU) == 0) {
1918 addConnectedNodes(SU, NewSet, NodesAdded);
1919 if (NewSet.size() > 0)
1920 NodeSets.push_back(NewSet);
1926 void SwingSchedulerDAG::addConnectedNodes(
SUnit *SU,
NodeSet &NewSet,
1930 for (
auto &SI : SU->
Succs) {
1932 if (!SI.isArtificial() && NodesAdded.
count(Successor) == 0)
1933 addConnectedNodes(Successor, NewSet, NodesAdded);
1935 for (
auto &PI : SU->
Preds) {
1936 SUnit *Predecessor = PI.getSUnit();
1937 if (!PI.isArtificial() && NodesAdded.
count(Predecessor) == 0)
1938 addConnectedNodes(Predecessor, NewSet, NodesAdded);
1947 for (
unsigned i = 0, e = Set1.
size(); i != e; ++
i) {
1949 if (Set2.count(SU) != 0)
1952 return !Result.
empty();
1956 void SwingSchedulerDAG::fuseRecs(NodeSetType &NodeSets) {
1957 for (NodeSetType::iterator I = NodeSets.
begin(), E = NodeSets.end(); I !=
E;
1960 for (NodeSetType::iterator J = I + 1; J !=
E;) {
1962 if (NI.getNode(0)->NodeNum == NJ.getNode(0)->NodeNum) {
1963 if (NJ.compareRecMII(NI) > 0)
1964 NI.setRecMII(NJ.getRecMII());
1965 for (NodeSet::iterator NII = J->begin(), ENI = J->end(); NII != ENI;
1978 void SwingSchedulerDAG::removeDuplicateNodes(NodeSetType &NodeSets) {
1979 for (NodeSetType::iterator I = NodeSets.
begin(), E = NodeSets.end(); I !=
E;
1981 for (NodeSetType::iterator J = I + 1; J !=
E;) {
1982 J->remove_if([&](
SUnit *SUJ) {
return I->
count(SUJ); });
1984 if (J->size() == 0) {
1995 for (
auto &SI : Inst1->
Succs)
1996 if (SI.getSUnit() == Inst2 && SI.getKind() ==
SDep::Data)
2005 void SwingSchedulerDAG::computeNodeOrder(NodeSetType &NodeSets) {
2009 for (
auto &Nodes : NodeSets) {
2010 DEBUG(
dbgs() <<
"NodeSet size " << Nodes.size() <<
"\n");
2025 DEBUG(
dbgs() <<
" Top down (intersect) ");
2026 }
else if (NodeSets.size() == 1) {
2027 for (
auto &N : Nodes)
2028 if (N->Succs.
size() == 0)
2034 SUnit *maxASAP =
nullptr;
2035 for (
SUnit *SU : Nodes) {
2036 if (maxASAP ==
nullptr || getASAP(SU) >= getASAP(maxASAP))
2041 DEBUG(
dbgs() <<
" Bottom up (default) ");
2044 while (!R.
empty()) {
2045 if (Order == TopDown) {
2049 while (!R.
empty()) {
2050 SUnit *maxHeight =
nullptr;
2051 for (
SUnit *I : R) {
2052 if (maxHeight ==
nullptr || getHeight(I) > getHeight(maxHeight))
2054 else if (getHeight(I) == getHeight(maxHeight) &&
2055 getMOV(I) < getMOV(maxHeight) &&
2063 R.remove(maxHeight);
2064 for (
const auto &I : maxHeight->
Succs) {
2065 if (Nodes.count(I.getSUnit()) == 0)
2071 R.insert(I.getSUnit());
2074 for (
const auto &I : maxHeight->
Preds) {
2077 if (Nodes.count(I.getSUnit()) == 0)
2081 R.insert(I.getSUnit());
2085 DEBUG(
dbgs() <<
"\n Switching order to bottom up ");
2093 while (!R.empty()) {
2094 SUnit *maxDepth =
nullptr;
2095 for (
SUnit *I : R) {
2096 if (maxDepth ==
nullptr || getDepth(I) > getDepth(maxDepth))
2098 else if (getDepth(I) == getDepth(maxDepth) &&
2099 getMOV(I) < getMOV(maxDepth) &&
2108 if (Nodes.isExceedSU(maxDepth)) {
2111 R.insert(Nodes.getNode(0));
2114 for (
const auto &I : maxDepth->
Preds) {
2115 if (Nodes.count(I.getSUnit()) == 0)
2121 R.insert(I.getSUnit());
2124 for (
const auto &I : maxDepth->
Succs) {
2127 if (Nodes.count(I.getSUnit()) == 0)
2131 R.insert(I.getSUnit());
2135 DEBUG(
dbgs() <<
"\n Switching order to top down ");
2141 DEBUG(
dbgs() <<
"\nDone with Nodeset\n");
2145 dbgs() <<
"Node order: ";
2147 dbgs() <<
" " << I->NodeNum <<
" ";
2154 bool SwingSchedulerDAG::schedulePipeline(SMSchedule &Schedule) {
2159 bool scheduleFound =
false;
2161 for (
unsigned II = MII; II < MII + 10 && !scheduleFound; ++II) {
2163 Schedule.setInitiationInterval(II);
2164 DEBUG(
dbgs() <<
"Try to schedule with " << II <<
"\n");
2173 int EarlyStart = INT_MIN;
2174 int LateStart = INT_MAX;
2177 int SchedEnd = INT_MAX;
2178 int SchedStart = INT_MIN;
2179 Schedule.computeStart(SU, &EarlyStart, &LateStart, &SchedEnd, &SchedStart,
2187 dbgs() <<
"\tes: " << EarlyStart <<
" ls: " << LateStart
2188 <<
" me: " << SchedEnd <<
" ms: " << SchedStart <<
"\n";
2191 if (EarlyStart > LateStart || SchedEnd < EarlyStart ||
2192 SchedStart > LateStart)
2193 scheduleFound =
false;
2194 else if (EarlyStart != INT_MIN && LateStart == INT_MAX) {
2195 SchedEnd =
std::min(SchedEnd, EarlyStart + (
int)II - 1);
2196 scheduleFound = Schedule.insert(SU, EarlyStart, SchedEnd, II);
2197 }
else if (EarlyStart == INT_MIN && LateStart != INT_MAX) {
2198 SchedStart = std::max(SchedStart, LateStart - (
int)II + 1);
2199 scheduleFound = Schedule.insert(SU, LateStart, SchedStart, II);
2200 }
else if (EarlyStart != INT_MIN && LateStart != INT_MAX) {
2207 scheduleFound = Schedule.insert(SU, SchedEnd, EarlyStart, II);
2209 scheduleFound = Schedule.insert(SU, EarlyStart, SchedEnd, II);
2211 int FirstCycle = Schedule.getFirstCycle();
2212 scheduleFound = Schedule.insert(SU, FirstCycle + getASAP(SU),
2213 FirstCycle + getASAP(SU) + II - 1, II);
2220 scheduleFound =
false;
2224 dbgs() <<
"\tCan't schedule\n";
2226 }
while (++NI != NE && scheduleFound);
2230 scheduleFound = Schedule.isValidSchedule(
this);
2233 DEBUG(
dbgs() <<
"Schedule Found? " << scheduleFound <<
"\n");
2236 Schedule.finalizeSchedule(
this);
2240 return scheduleFound && Schedule.getMaxStageCount() > 0;
2248 void SwingSchedulerDAG::generatePipelinedLoop(SMSchedule &Schedule) {
2252 unsigned MaxStageCount = Schedule.getMaxStageCount();
2258 ValueMapTy *VRMap =
new ValueMapTy[(MaxStageCount + 1) * 2];
2259 InstrMapTy InstrMap;
2263 generateProlog(Schedule, MaxStageCount, KernelBB, VRMap, PrologBBs);
2264 MF.insert(BB->getIterator(), KernelBB);
2268 for (
int Cycle = Schedule.getFirstCycle(),
2269 LastCycle = Schedule.getFinalCycle();
2270 Cycle <= LastCycle; ++Cycle) {
2271 std::deque<SUnit *> &CycleInstrs = Schedule.getInstructions(Cycle);
2273 for (
SUnit *CI : CycleInstrs) {
2274 if (CI->getInstr()->isPHI())
2276 unsigned StageNum = Schedule.stageScheduled(getSUnit(CI->getInstr()));
2277 MachineInstr *NewMI = cloneInstr(CI->getInstr(), MaxStageCount, StageNum);
2278 updateInstruction(NewMI,
false, MaxStageCount, StageNum, Schedule, VRMap);
2280 InstrMap[NewMI] = CI->getInstr();
2287 E = BB->instr_end();
2290 updateInstruction(NewMI,
false, MaxStageCount, 0, Schedule, VRMap);
2292 InstrMap[NewMI] = &*
I;
2298 generateExistingPhis(KernelBB, PrologBBs.
back(), KernelBB, KernelBB, Schedule,
2299 VRMap, InstrMap, MaxStageCount, MaxStageCount,
false);
2300 generatePhis(KernelBB, PrologBBs.
back(), KernelBB, KernelBB, Schedule, VRMap,
2301 InstrMap, MaxStageCount, MaxStageCount,
false);
2303 DEBUG(
dbgs() <<
"New block\n"; KernelBB->dump(););
2307 generateEpilog(Schedule, MaxStageCount, KernelBB, VRMap, EpilogBBs,
2312 splitLifetimes(KernelBB, EpilogBBs, Schedule);
2315 removeDeadInstructions(KernelBB, EpilogBBs);
2318 addBranches(PrologBBs, KernelBB, EpilogBBs, Schedule, VRMap);
2322 BB->eraseFromParent();
2328 void SwingSchedulerDAG::generateProlog(SMSchedule &Schedule,
unsigned LastStage,
2331 MBBVectorTy &PrologBBs) {
2333 assert(PreheaderBB != NULL &&
2334 "Need to add code to handle loops w/o preheader");
2336 InstrMapTy InstrMap;
2341 for (
unsigned i = 0; i < LastStage; ++
i) {
2346 MF.insert(BB->getIterator(), NewBB);
2353 for (
int StageNum = i; StageNum >= 0; --StageNum) {
2355 BBE = BB->getFirstTerminator();
2356 BBI != BBE; ++BBI) {
2357 if (Schedule.isScheduledAtStage(getSUnit(&*BBI), (
unsigned)StageNum)) {
2361 cloneAndChangeInstr(&*BBI, i, (
unsigned)StageNum, Schedule);
2362 updateInstruction(NewMI,
false, i, (
unsigned)StageNum, Schedule,
2365 InstrMap[NewMI] = &*BBI;
2369 rewritePhiValues(NewBB, i, Schedule, VRMap, InstrMap);
2371 dbgs() <<
"prolog:\n";
2390 void SwingSchedulerDAG::generateEpilog(SMSchedule &Schedule,
unsigned LastStage,
2393 MBBVectorTy &EpilogBBs,
2394 MBBVectorTy &PrologBBs) {
2400 assert(!checkBranch &&
"generateEpilog must be able to analyze the branch");
2405 if (*LoopExitI == KernelBB)
2407 assert(LoopExitI != KernelBB->
succ_end() &&
"Expecting a successor");
2412 InstrMapTy InstrMap;
2417 int EpilogStage = LastStage + 1;
2418 for (
unsigned i = LastStage; i >= 1; --
i, ++EpilogStage) {
2421 MF.insert(BB->getIterator(), NewBB);
2426 if (EpilogStart == LoopExitBB)
2427 EpilogStart = NewBB;
2431 for (
unsigned StageNum = i; StageNum <= LastStage; ++StageNum) {
2432 for (
auto &BBI : *BB) {
2436 if (Schedule.isScheduledAtStage(getSUnit(In), StageNum)) {
2437 MachineInstr *NewMI = cloneInstr(In, EpilogStage - LastStage, 0);
2438 updateInstruction(NewMI, i == 1, EpilogStage, 0, Schedule, VRMap);
2440 InstrMap[NewMI] =
In;
2444 generateExistingPhis(NewBB, PrologBBs[i - 1], PredBB, KernelBB, Schedule,
2445 VRMap, InstrMap, LastStage, EpilogStage, i == 1);
2446 generatePhis(NewBB, PrologBBs[i - 1], PredBB, KernelBB, Schedule, VRMap,
2447 InstrMap, LastStage, EpilogStage, i == 1);
2451 dbgs() <<
"epilog:\n";
2460 for (
unsigned i = 2, e = MI.
getNumOperands() + 1; i != e; i += 2) {
2472 if (EpilogBBs.size() > 0) {
2504 if (I->getParent()->getParent() != BB)
2512 void SwingSchedulerDAG::generateExistingPhis(
2515 InstrMapTy &InstrMap,
unsigned LastStageNum,
unsigned CurStageNum,
2520 unsigned PrologStage = 0;
2521 unsigned PrevStage = 0;
2522 bool InKernel = (LastStageNum == CurStageNum);
2524 PrologStage = LastStageNum - 1;
2525 PrevStage = CurStageNum;
2527 PrologStage = LastStageNum - (CurStageNum - LastStageNum);
2528 PrevStage = LastStageNum + (CurStageNum - LastStageNum) - 1;
2532 BBE = BB->getFirstNonPHI();
2533 BBI != BBE; ++BBI) {
2534 unsigned Def = BBI->getOperand(0).getReg();
2536 unsigned InitVal = 0;
2537 unsigned LoopVal = 0;
2540 unsigned PhiOp1 = 0;
2543 unsigned PhiOp2 = LoopVal;
2544 if (VRMap[LastStageNum].
count(LoopVal))
2545 PhiOp2 = VRMap[LastStageNum][LoopVal];
2547 int StageScheduled = Schedule.stageScheduled(getSUnit(&*BBI));
2549 Schedule.stageScheduled(getSUnit(
MRI.getVRegDef(LoopVal)));
2550 unsigned NumStages = Schedule.getStagesForReg(Def, CurStageNum);
2551 if (NumStages == 0) {
2554 unsigned NewReg = VRMap[PrevStage][LoopVal];
2555 rewriteScheduledInstr(NewBB, Schedule, InstrMap, CurStageNum, 0, &*BBI,
2557 if (VRMap[CurStageNum].
count(LoopVal))
2558 VRMap[CurStageNum][
Def] = VRMap[CurStageNum][LoopVal];
2562 unsigned NumPhis = NumStages;
2563 if (!InKernel && (
int)PrologStage < LoopValStage)
2567 NumPhis = std::max((
int)NumPhis - (
int)(LoopValStage - PrologStage), 1);
2570 NumPhis =
std::min(NumPhis, PrologStage + 2);
2572 unsigned NewReg = 0;
2574 unsigned AccessStage = (LoopValStage != -1) ? LoopValStage : StageScheduled;
2581 if (!InKernel && StageScheduled >= LoopValStage && AccessStage == 0 &&
2586 if (InKernel && LoopValStage != -1 && StageScheduled > LoopValStage)
2587 StageDiff = StageScheduled - LoopValStage;
2588 for (
unsigned np = 0; np < NumPhis; ++np) {
2592 if (np > PrologStage || StageScheduled >= (
int)LastStageNum)
2595 else if (PrologStage >= AccessStage + StageDiff + np &&
2596 VRMap[PrologStage - StageDiff - np].
count(LoopVal) != 0)
2597 PhiOp1 = VRMap[PrologStage - StageDiff - np][LoopVal];
2600 else if (PrologStage >= AccessStage + StageDiff + np) {
2606 while (InstOp1 && InstOp1->
isPHI() && InstOp1->
getParent() == BB) {
2607 int PhiStage = Schedule.stageScheduled(getSUnit(InstOp1));
2608 if ((
int)(PrologStage - StageDiff - np) < PhiStage + Indirects)
2612 InstOp1 =
MRI.getVRegDef(PhiOp1);
2613 int PhiOpStage = Schedule.stageScheduled(getSUnit(InstOp1));
2614 int StageAdj = (PhiOpStage != -1 ? PhiStage - PhiOpStage : 0);
2615 if (PhiOpStage != -1 && PrologStage - StageAdj >= Indirects + np &&
2616 VRMap[PrologStage - StageAdj - Indirects - np].
count(PhiOp1)) {
2617 PhiOp1 = VRMap[PrologStage - StageAdj - Indirects - np][PhiOp1];
2631 bool LoopDefIsPhi = PhiInst && PhiInst->
isPHI();
2636 int StageDiffAdj = 0;
2637 if (LoopValStage != -1 && StageScheduled > LoopValStage)
2638 StageDiffAdj = StageScheduled - LoopValStage;
2641 if (np == 0 && PrevStage == LastStageNum &&
2642 (StageScheduled != 0 || LoopValStage != 0) &&
2643 VRMap[PrevStage - StageDiffAdj].
count(LoopVal))
2644 PhiOp2 = VRMap[PrevStage - StageDiffAdj][LoopVal];
2647 else if (np > 0 && PrevStage == LastStageNum &&
2648 VRMap[PrevStage - np + 1].
count(Def))
2649 PhiOp2 = VRMap[PrevStage - np + 1][
Def];
2651 else if ((
unsigned)LoopValStage + StageDiffAdj > PrologStage + 1 &&
2652 VRMap[PrevStage - StageDiffAdj - np].
count(LoopVal))
2653 PhiOp2 = VRMap[PrevStage - StageDiffAdj - np][LoopVal];
2656 else if (VRMap[PrevStage - np].
count(Def) &&
2657 (!LoopDefIsPhi || PrevStage != LastStageNum))
2658 PhiOp2 = VRMap[PrevStage - np][
Def];
2665 if (LoopDefIsPhi && (
int)PrologStage >= StageScheduled) {
2666 int LVNumStages = Schedule.getStagesForPhi(LoopVal);
2667 int StageDiff = (StageScheduled - LoopValStage);
2668 LVNumStages -= StageDiff;
2669 if (LVNumStages > (
int)np) {
2671 unsigned ReuseStage = CurStageNum;
2672 if (Schedule.isLoopCarried(
this, *PhiInst))
2673 ReuseStage -= LVNumStages;
2676 if (VRMap[ReuseStage].
count(LoopVal)) {
2677 NewReg = VRMap[ReuseStage][LoopVal];
2679 rewriteScheduledInstr(NewBB, Schedule, InstrMap, CurStageNum, np,
2680 &*BBI, Def, NewReg);
2682 VRMap[CurStageNum - np][
Def] = NewReg;
2684 if (VRMap[LastStageNum - np - 1].
count(LoopVal))
2685 PhiOp2 = VRMap[LastStageNum - np - 1][LoopVal];
2687 if (IsLast && np == NumPhis - 1)
2691 }
else if (InKernel && StageDiff > 0 &&
2692 VRMap[CurStageNum - StageDiff - np].
count(LoopVal))
2693 PhiOp2 = VRMap[CurStageNum - StageDiff - np][LoopVal];
2697 NewReg =
MRI.createVirtualRegister(RC);
2701 TII->get(TargetOpcode::PHI), NewReg);
2705 InstrMap[NewPhi] = &*BBI;
2710 unsigned PrevReg = 0;
2711 if (InKernel && VRMap[PrevStage - np].
count(LoopVal))
2712 PrevReg = VRMap[PrevStage - np][LoopVal];
2713 rewriteScheduledInstr(NewBB, Schedule, InstrMap, CurStageNum, np, &*BBI,
2714 Def, NewReg, PrevReg);
2716 if (VRMap[CurStageNum - np].
count(Def)) {
2717 unsigned R = VRMap[CurStageNum - np][
Def];
2718 rewriteScheduledInstr(NewBB, Schedule, InstrMap, CurStageNum, np, &*BBI,
2725 if (IsLast && np == NumPhis - 1)
2733 VRMap[CurStageNum - np][
Def] = NewReg;
2736 while (NumPhis++ < NumStages) {
2737 rewriteScheduledInstr(NewBB, Schedule, InstrMap, CurStageNum, NumPhis,
2738 &*BBI, Def, NewReg, 0);
2743 if (NumStages == 0 && IsLast && VRMap[CurStageNum].
count(LoopVal))
2751 void SwingSchedulerDAG::generatePhis(
2754 InstrMapTy &InstrMap,
unsigned LastStageNum,
unsigned CurStageNum,
2758 unsigned PrologStage = 0;
2759 unsigned PrevStage = 0;
2760 unsigned StageDiff = CurStageNum - LastStageNum;
2761 bool InKernel = (StageDiff == 0);
2763 PrologStage = LastStageNum - 1;
2764 PrevStage = CurStageNum;
2766 PrologStage = LastStageNum - StageDiff;
2767 PrevStage = LastStageNum + StageDiff - 1;
2771 BBE = BB->instr_end();
2772 BBI != BBE; ++BBI) {
2773 for (
unsigned i = 0, e = BBI->getNumOperands(); i != e; ++
i) {
2779 int StageScheduled = Schedule.stageScheduled(getSUnit(&*BBI));
2780 assert(StageScheduled != -1 &&
"Expecting scheduled instruction.");
2781 unsigned Def = MO.
getReg();
2782 unsigned NumPhis = Schedule.getStagesForReg(Def, CurStageNum);
2786 if (!InKernel && NumPhis == 0 && StageScheduled == 0 &&
2789 if (!InKernel && (
unsigned)StageScheduled > PrologStage)
2792 unsigned PhiOp2 = VRMap[PrevStage][
Def];
2794 if (InstOp2->isPHI() && InstOp2->getParent() == NewBB)
2798 if (NumPhis > PrologStage + 1 - StageScheduled)
2799 NumPhis = PrologStage + 1 - StageScheduled;
2800 for (
unsigned np = 0; np < NumPhis; ++np) {
2801 unsigned PhiOp1 = VRMap[PrologStage][
Def];
2802 if (np <= PrologStage)
2803 PhiOp1 = VRMap[PrologStage - np][
Def];
2811 PhiOp2 = VRMap[PrevStage - np][
Def];
2814 unsigned NewReg =
MRI.createVirtualRegister(RC);
2818 TII->get(TargetOpcode::PHI), NewReg);
2822 InstrMap[NewPhi] = &*BBI;
2827 rewriteScheduledInstr(NewBB, Schedule, InstrMap, CurStageNum, np,
2828 &*BBI, PhiOp1, NewReg);
2829 rewriteScheduledInstr(NewBB, Schedule, InstrMap, CurStageNum, np,
2830 &*BBI, PhiOp2, NewReg);
2833 VRMap[PrevStage - np - 1][
Def] = NewReg;
2835 VRMap[CurStageNum - np][
Def] = NewReg;
2836 if (np == NumPhis - 1)
2837 rewriteScheduledInstr(NewBB, Schedule, InstrMap, CurStageNum, np,
2838 &*BBI, Def, NewReg);
2840 if (IsLast && np == NumPhis - 1)
2852 MBBVectorTy &EpilogBBs) {
2855 for (MBBVectorTy::reverse_iterator MBB = EpilogBBs.
rbegin(),
2856 MBE = EpilogBBs.rend();
2859 ME = (*MBB)->instr_rend();
2876 MOI != MOE; ++MOI) {
2877 if (!MOI->isReg() || !MOI->isDef())
2879 unsigned reg = MOI->getReg();
2880 unsigned realUses = 0;
2886 if (UI->getParent()->getParent() != BB) {
2910 if (
MRI.use_begin(reg) ==
MRI.use_end()) {
2927 MBBVectorTy &EpilogBBs,
2928 SMSchedule &Schedule) {
2932 BBI != BBF; ++BBI) {
2933 unsigned Def = BBI->getOperand(0).getReg();
2937 E =
MRI.use_instr_end();
2939 if (I->isPHI() && I->getParent() == KernelBB) {
2949 unsigned SplitReg = 0;
2952 if (BBJ.readsRegister(Def)) {
2954 if (SplitReg == 0) {
2955 SplitReg =
MRI.createVirtualRegister(
MRI.getRegClass(Def));
2957 TII->get(TargetOpcode::COPY), SplitReg)
2960 BBJ.substituteRegister(Def, SplitReg, 0, *TRI);
2965 for (
auto &Epilog : EpilogBBs)
2966 for (
auto &I : *Epilog)
2967 if (I.readsRegister(Def))
2968 I.substituteRegister(Def, SplitReg, 0, *TRI);
2992 void SwingSchedulerDAG::addBranches(MBBVectorTy &PrologBBs,
2994 MBBVectorTy &EpilogBBs,
2995 SMSchedule &Schedule, ValueMapTy *VRMap) {
2996 assert(PrologBBs.size() == EpilogBBs.size() &&
"Prolog/Epilog mismatch");
3005 unsigned MaxIter = PrologBBs.size() - 1;
3006 unsigned LC = UINT_MAX;
3007 unsigned LCMin = UINT_MAX;
3008 for (
unsigned i = 0, j = MaxIter; i <=
MaxIter; ++
i, --j) {
3024 if (LCMin == UINT_MAX)
3027 unsigned numAdded = 0;
3031 }
else if (j >= LCMin) {
3038 if (LastPro != LastEpi) {
3052 I != E && numAdded > 0; ++
I, --numAdded)
3053 updateInstruction(&*I,
false, j, 0, Schedule, VRMap);
3069 if (BaseDef && BaseDef->
isPHI()) {
3087 void SwingSchedulerDAG::updateMemOperands(
MachineInstr &NewMI,
3099 if (MMO->isVolatile() || (MMO->isInvariant() && MMO->isDereferenceable()) ||
3100 (!MMO->getValue())) {
3101 NewMemRefs[Refs++] = MMO;
3106 int64_t AdjOffset = Delta * Num;
3107 NewMemRefs[Refs++] =
3108 MF.getMachineMemOperand(MMO, AdjOffset, MMO->getSize());
3110 NewMemRefs[Refs++] = MF.getMachineMemOperand(MMO, 0, UINT64_MAX);
3112 NewMI.
setMemRefs(NewMemRefs, NewMemRefs + NumRefs);
3118 unsigned CurStageNum,
3119 unsigned InstStageNum) {
3132 updateMemOperands(*NewMI, *OldMI, CurStageNum - InstStageNum);
3140 unsigned CurStageNum,
3141 unsigned InstStageNum,
3142 SMSchedule &Schedule) {
3145 InstrChanges.
find(getSUnit(OldMI));
3146 if (It != InstrChanges.
end()) {
3147 std::pair<unsigned, int64_t> RegAndOffset = It->second;
3148 unsigned BasePos, OffsetPos;
3152 MachineInstr *LoopDef = findDefInLoop(RegAndOffset.first);
3153 if (Schedule.stageScheduled(getSUnit(LoopDef)) > (signed)InstStageNum)
3154 NewOffset += RegAndOffset.second * (CurStageNum - InstStageNum);
3157 updateMemOperands(*NewMI, *OldMI, CurStageNum - InstStageNum);
3163 void SwingSchedulerDAG::updateInstruction(
MachineInstr *NewMI,
bool LastDef,
3164 unsigned CurStageNum,
3165 unsigned InstrStageNum,
3166 SMSchedule &Schedule,
3167 ValueMapTy *VRMap) {
3172 unsigned reg = MO.
getReg();
3178 VRMap[CurStageNum][reg] = NewReg;
3181 }
else if (MO.
isUse()) {
3184 int DefStageNum = Schedule.stageScheduled(getSUnit(Def));
3185 unsigned StageNum = CurStageNum;
3186 if (DefStageNum != -1 && (
int)InstrStageNum > DefStageNum) {
3188 unsigned StageDiff = (InstrStageNum - DefStageNum);
3190 StageNum -= StageDiff;
3192 if (VRMap[StageNum].
count(reg))
3193 MO.
setReg(VRMap[StageNum][reg]);
3201 MachineInstr *SwingSchedulerDAG::findDefInLoop(
unsigned Reg) {
3204 while (Def->
isPHI()) {
3205 if (!Visited.
insert(Def).second)
3217 unsigned SwingSchedulerDAG::getPrevMapVal(
unsigned StageNum,
unsigned PhiStage,
3218 unsigned LoopVal,
unsigned LoopStage,
3221 unsigned PrevVal = 0;
3222 if (StageNum > PhiStage) {
3224 if (PhiStage == LoopStage && VRMap[StageNum - 1].
count(LoopVal))
3226 PrevVal = VRMap[StageNum - 1][LoopVal];
3227 else if (VRMap[StageNum].
count(LoopVal))
3230 PrevVal = VRMap[StageNum][LoopVal];
3234 else if (StageNum == PhiStage + 1)
3237 else if (StageNum > PhiStage + 1 && LoopInst->
getParent() == BB)
3240 getPrevMapVal(StageNum - 1, PhiStage,
getLoopPhiReg(*LoopInst, BB),
3241 LoopStage, VRMap, BB);
3252 SMSchedule &Schedule,
3254 InstrMapTy &InstrMap) {
3257 BBI != BBE; ++BBI) {
3258 unsigned InitVal = 0;
3259 unsigned LoopVal = 0;
3261 unsigned PhiDef = BBI->getOperand(0).getReg();
3265 unsigned LoopStage =
3267 unsigned NumPhis = Schedule.getStagesForPhi(PhiDef);
3268 if (NumPhis > StageNum)
3270 for (
unsigned np = 0; np <= NumPhis; ++np) {
3272 getPrevMapVal(StageNum - np, PhiStage, LoopVal, LoopStage, VRMap, BB);
3275 rewriteScheduledInstr(NewBB, Schedule, InstrMap, StageNum - np, np, &*BBI,
3284 void SwingSchedulerDAG::rewriteScheduledInstr(
3286 unsigned CurStageNum,
unsigned PhiNum,
MachineInstr *Phi,
unsigned OldReg,
3287 unsigned NewReg,
unsigned PrevReg) {
3288 bool InProlog = (CurStageNum < Schedule.getMaxStageCount());
3289 int StagePhi = Schedule.stageScheduled(getSUnit(Phi)) + PhiNum;
3300 if (UseMI->
isPHI()) {
3306 InstrMapTy::iterator OrigInstr = InstrMap.find(UseMI);
3307 assert(OrigInstr != InstrMap.end() &&
"Instruction not scheduled.");
3308 SUnit *OrigMISU = getSUnit(OrigInstr->second);
3309 int StageSched = Schedule.stageScheduled(OrigMISU);
3310 int CycleSched = Schedule.cycleScheduled(OrigMISU);
3311 unsigned ReplaceReg = 0;
3313 if (StagePhi == StageSched && Phi->
isPHI()) {
3314 int CyclePhi = Schedule.cycleScheduled(getSUnit(Phi));
3315 if (PrevReg && InProlog)
3316 ReplaceReg = PrevReg;
3317 else if (PrevReg && !Schedule.isLoopCarried(
this, *Phi) &&
3318 (CyclePhi <= CycleSched || OrigMISU->getInstr()->isPHI()))
3319 ReplaceReg = PrevReg;
3321 ReplaceReg = NewReg;
3325 if (!InProlog && StagePhi + 1 == StageSched &&
3326 !Schedule.isLoopCarried(
this, *Phi))
3327 ReplaceReg = NewReg;
3328 if (StagePhi > StageSched && Phi->
isPHI())
3329 ReplaceReg = NewReg;
3330 if (!InProlog && !Phi->
isPHI() && StagePhi < StageSched)
3331 ReplaceReg = NewReg;
3334 UseOp.
setReg(ReplaceReg);
3346 bool SwingSchedulerDAG::canUseLastOffsetValue(
MachineInstr *MI,
3348 unsigned &OffsetPos,
3354 unsigned BasePosLd, OffsetPosLd;
3362 if (!Phi || !Phi->
isPHI())
3371 if (!PrevDef || PrevDef == MI)
3377 unsigned BasePos1 = 0, OffsetPos1 = 0;
3384 if ((LoadOffset >= 0) != (StoreOffset >= 0))
3388 BasePos = BasePosLd;
3389 OffsetPos = OffsetPosLd;
3391 Offset = StoreOffset;
3398 SMSchedule &Schedule,
3400 SUnit *SU = getSUnit(MI);
3402 InstrChanges.
find(SU);
3403 if (It != InstrChanges.
end()) {
3404 std::pair<unsigned, int64_t> RegAndOffset = It->second;
3405 unsigned BasePos, OffsetPos;
3410 int DefStageNum = Schedule.stageScheduled(getSUnit(LoopDef));
3411 int DefCycleNum = Schedule.cycleScheduled(getSUnit(LoopDef));
3412 int BaseStageNum = Schedule.stageScheduled(SU);
3413 int BaseCycleNum = Schedule.cycleScheduled(SU);
3414 if (BaseStageNum < DefStageNum) {
3416 int OffsetDiff = DefStageNum - BaseStageNum;
3417 if (DefCycleNum < BaseCycleNum) {
3427 MISUnitMap[NewMI] = SU;
3429 NewMIs.insert(NewMI);
3439 bool SwingSchedulerDAG::isLoopCarriedOrder(
SUnit *Source,
const SDep &Dep,
3451 assert(SI !=
nullptr && DI !=
nullptr &&
"Expecting SUnit with an MI.");
3462 unsigned DeltaS, DeltaD;
3466 unsigned BaseRegS, BaseRegD;
3467 int64_t OffsetS, OffsetD;
3473 if (BaseRegS != BaseRegD)
3481 if (OffsetS >= OffsetD)
3482 return OffsetS + AccessSizeS > DeltaS;
3483 else if (OffsetS < OffsetD)
3484 return OffsetD + AccessSizeD > DeltaD;
3489 void SwingSchedulerDAG::postprocessDAG() {
3490 for (
auto &M : Mutations)
3499 bool SMSchedule::insert(
SUnit *SU,
int StartCycle,
int EndCycle,
int II) {
3500 bool forward =
true;
3501 if (StartCycle > EndCycle)
3505 int termCycle = forward ? EndCycle + 1 : EndCycle - 1;
3506 for (
int curCycle = StartCycle; curCycle != termCycle;
3507 forward ? ++curCycle : --curCycle) {
3510 Resources->clearResources();
3511 for (
int checkCycle = FirstCycle + ((curCycle - FirstCycle) % II);
3512 checkCycle <= LastCycle; checkCycle += II) {
3513 std::deque<SUnit *> &cycleInstrs = ScheduledInstrs[checkCycle];
3515 for (std::deque<SUnit *>::iterator I = cycleInstrs.
begin(),
3516 E = cycleInstrs.end();
3520 assert(Resources->canReserveResources(*(*I)->getInstr()) &&
3521 "These instructions have already been scheduled.");
3522 Resources->reserveResources(*(*I)->getInstr());
3526 Resources->canReserveResources(*SU->
getInstr())) {
3528 dbgs() <<
"\tinsert at cycle " << curCycle <<
" ";
3532 ScheduledInstrs[curCycle].push_back(SU);
3533 InstrToCycle.insert(std::make_pair(SU, curCycle));
3534 if (curCycle > LastCycle)
3535 LastCycle = curCycle;
3536 if (curCycle < FirstCycle)
3537 FirstCycle = curCycle;
3541 dbgs() <<
"\tfailed to insert at cycle " << curCycle <<
" ";
3549 int SMSchedule::earliestCycleInChain(
const SDep &Dep) {
3553 int EarlyCycle = INT_MAX;
3554 while (!Worklist.
empty()) {
3557 if (Visited.
count(PrevSU))
3559 std::map<SUnit *, int>::const_iterator it = InstrToCycle.find(PrevSU);
3560 if (it == InstrToCycle.end())
3562 EarlyCycle =
std::min(EarlyCycle, it->second);
3563 for (
const auto &PI : PrevSU->
Preds)
3564 if (SwingSchedulerDAG::isOrder(PrevSU, PI))
3572 int SMSchedule::latestCycleInChain(
const SDep &Dep) {
3576 int LateCycle = INT_MIN;
3577 while (!Worklist.
empty()) {
3580 if (Visited.
count(SuccSU))
3582 std::map<SUnit *, int>::const_iterator it = InstrToCycle.find(SuccSU);
3583 if (it == InstrToCycle.end())
3585 LateCycle = std::max(LateCycle, it->second);
3586 for (
const auto &SI : SuccSU->
Succs)
3587 if (SwingSchedulerDAG::isOrder(SuccSU, SI))
3598 for (
auto &P : SU->
Preds)
3599 if (DAG->isBackedge(SU, P) && P.getSUnit()->getInstr()->isPHI())
3600 for (
auto &S : P.getSUnit()->Succs)
3601 if (S.getKind() ==
SDep::Order && S.getSUnit()->getInstr()->isPHI())
3602 return P.getSUnit();
3608 void SMSchedule::computeStart(
SUnit *SU,
int *MaxEarlyStart,
int *MinLateStart,
3609 int *MinEnd,
int *MaxStart,
int II,
3610 SwingSchedulerDAG *DAG) {
3614 for (
int cycle = getFirstCycle(); cycle <= LastCycle; ++cycle) {
3617 for (
SUnit *I : getInstructions(cycle)) {
3620 for (
unsigned i = 0, e = (
unsigned)SU->
Preds.size(); i != e; ++
i) {
3623 if (!DAG->isBackedge(SU, Dep)) {
3624 int EarlyStart = cycle + DAG->getLatency(SU, Dep) -
3625 DAG->getDistance(Dep.
getSUnit(), SU, Dep) * II;
3626 *MaxEarlyStart = std::max(*MaxEarlyStart, EarlyStart);
3627 if (DAG->isLoopCarriedOrder(SU, Dep,
false)) {
3628 int End = earliestCycleInChain(Dep) + (II - 1);
3632 int LateStart = cycle - DAG->getLatency(SU, Dep) +
3633 DAG->getDistance(SU, Dep.
getSUnit(), Dep) * II;
3634 *MinLateStart =
std::min(*MinLateStart, LateStart);
3642 *MinLateStart =
std::min(*MinLateStart, cycle);
3644 for (
unsigned i = 0, e = (
unsigned)SU->
Succs.size(); i != e; ++
i)
3645 if (SU->
Succs[i].getSUnit() ==
I) {
3647 if (!DAG->isBackedge(SU, Dep)) {
3648 int LateStart = cycle - DAG->getLatency(SU, Dep) +
3649 DAG->getDistance(SU, Dep.
getSUnit(), Dep) * II;
3650 *MinLateStart =
std::min(*MinLateStart, LateStart);
3651 if (DAG->isLoopCarriedOrder(SU, Dep)) {
3652 int Start = latestCycleInChain(Dep) + 1 - II;
3653 *MaxStart = std::max(*MaxStart, Start);
3656 int EarlyStart = cycle + DAG->getLatency(SU, Dep) -
3657 DAG->getDistance(Dep.
getSUnit(), SU, Dep) * II;
3658 *MaxEarlyStart = std::max(*MaxEarlyStart, EarlyStart);
3668 bool SMSchedule::orderDependence(SwingSchedulerDAG *SSD,
SUnit *SU,
3669 std::deque<SUnit *> &Insts) {
3671 bool OrderBeforeUse =
false;
3672 bool OrderAfterDef =
false;
3673 bool OrderBeforeDef =
false;
3674 unsigned MoveDef = 0;
3675 unsigned MoveUse = 0;
3676 int StageInst1 = stageScheduled(SU);
3679 for (std::deque<SUnit *>::iterator I = Insts.
begin(), E = Insts.end(); I !=
E;
3682 if (MI->
isPHI() && (*I)->getInstr()->isPHI())
3688 unsigned Reg = MO.
getReg();
3689 unsigned BasePos, OffsetPos;
3692 if (
unsigned NewReg = SSD->getInstrBaseReg(SU))
3695 std::tie(Reads, Writes) =
3696 (*I)->getInstr()->readsWritesVirtualRegister(Reg);
3697 if (MO.
isDef() && Reads && stageScheduled(*I) <= StageInst1) {
3698 OrderBeforeUse =
true;
3700 }
else if (MO.
isDef() && Reads && stageScheduled(*I) > StageInst1) {
3702 OrderAfterDef =
true;
3704 }
else if (MO.
isUse() && Writes && stageScheduled(*I) == StageInst1) {
3705 if (cycleScheduled(*I) == cycleScheduled(SU) && !(*I)->isSucc(SU)) {
3706 OrderBeforeUse =
true;
3709 OrderAfterDef =
true;
3712 }
else if (MO.
isUse() && Writes && stageScheduled(*I) > StageInst1) {
3713 OrderBeforeUse =
true;
3716 OrderAfterDef =
true;
3719 }
else if (MO.
isUse() && Writes && stageScheduled(*I) < StageInst1) {
3721 OrderBeforeUse =
true;
3723 }
else if (MO.
isUse() && stageScheduled(*I) == StageInst1 &&
3724 isLoopCarriedDefOfUse(SSD, (*I)->getInstr(), MO)) {
3725 OrderBeforeDef =
true;
3731 for (
auto &S : SU->
Succs)
3733 if (S.getSUnit() == *I && stageScheduled(*I) == StageInst1) {
3734 OrderBeforeUse =
true;
3738 if (cycleScheduled(SU) != cycleScheduled(S.getSUnit())) {
3739 if (S.isAssignedRegDep()) {
3740 OrderAfterDef =
true;
3744 OrderBeforeUse =
true;
3748 for (
auto &P : SU->
Preds)
3750 if (P.getSUnit() == *I && stageScheduled(*I) == StageInst1) {
3751 OrderAfterDef =
true;
3755 if (cycleScheduled(SU) != cycleScheduled(P.getSUnit())) {
3756 if (P.isAssignedRegDep()) {
3757 OrderBeforeUse =
true;
3761 OrderAfterDef =
true;
3768 if (OrderAfterDef && OrderBeforeUse && MoveUse == MoveDef)
3769 OrderBeforeUse =
false;
3774 OrderBeforeUse = !OrderAfterDef || (MoveUse > MoveDef);
3778 if (OrderBeforeUse && OrderAfterDef) {
3779 SUnit *UseSU = Insts.at(MoveUse);
3780 SUnit *DefSU = Insts.at(MoveDef);
3781 if (MoveUse > MoveDef) {
3782 Insts.erase(Insts.begin() + MoveUse);
3783 Insts.erase(Insts.begin() + MoveDef);
3785 Insts.erase(Insts.begin() + MoveDef);
3786 Insts.erase(Insts.begin() + MoveUse);
3788 if (orderDependence(SSD, UseSU, Insts)) {
3789 Insts.push_front(SU);
3790 orderDependence(SSD, DefSU, Insts);
3794 Insts.push_back(SU);
3795 Insts.push_back(UseSU);
3796 orderDependence(SSD, DefSU, Insts);
3802 Insts.push_front(SU);
3804 Insts.push_back(SU);
3805 return OrderBeforeUse;
3809 bool SMSchedule::isLoopCarried(SwingSchedulerDAG *SSD,
MachineInstr &Phi) {
3813 SUnit *DefSU = SSD->getSUnit(&Phi);
3814 unsigned DefCycle = cycleScheduled(DefSU);
3815 int DefStage = stageScheduled(DefSU);
3817 unsigned InitVal = 0;
3818 unsigned LoopVal = 0;
3823 if (UseSU->getInstr()->isPHI())
3825 unsigned LoopCycle = cycleScheduled(UseSU);
3826 int LoopStage = stageScheduled(UseSU);
3827 return (LoopCycle > DefCycle) || (LoopStage <= DefStage);
3837 bool SMSchedule::isLoopCarriedDefOfUse(SwingSchedulerDAG *SSD,
3846 if (!isLoopCarried(SSD, *Phi))
3853 if (DMO.
getReg() == LoopReg)
3863 bool SMSchedule::isValidSchedule(SwingSchedulerDAG *SSD) {
3864 for (
int i = 0, e = SSD->SUnits.size(); i < e; ++
i) {
3865 SUnit &SU = SSD->SUnits[
i];
3868 int StageDef = stageScheduled(&SU);
3869 assert(StageDef != -1 &&
"Instruction should have been scheduled.");
3870 for (
auto &SI : SU.
Succs)
3871 if (SI.isAssignedRegDep())
3873 if (stageScheduled(SI.getSUnit()) != StageDef)
3882 void SMSchedule::finalizeSchedule(SwingSchedulerDAG *SSD) {
3884 for (
int cycle = getFirstCycle(); cycle <= getFinalCycle(); ++cycle) {
3885 for (
int stage = 1, lastStage = getMaxStageCount(); stage <= lastStage;
3887 std::deque<SUnit *> &cycleInstrs =
3888 ScheduledInstrs[cycle + (stage * InitiationInterval)];
3889 for (std::deque<SUnit *>::reverse_iterator I = cycleInstrs.
rbegin(),
3890 E = cycleInstrs.rend();
3892 ScheduledInstrs[cycle].push_front(*I);
3897 for (
auto &I : InstrToCycle) {
3898 int DefStage = stageScheduled(I.first);
3905 unsigned Reg = Op.
getReg();
3906 unsigned MaxDiff = 0;
3907 bool PhiIsSwapped =
false;
3913 SUnit *SUnitUse = SSD->getSUnit(UseMI);
3914 int UseStage = stageScheduled(SUnitUse);
3916 if (UseStage != -1 && UseStage >= DefStage)
3917 Diff = UseStage - DefStage;
3919 if (isLoopCarried(SSD, *MI))
3922 PhiIsSwapped =
true;
3924 MaxDiff = std::max(Diff, MaxDiff);
3926 RegToStageDiff[
Reg] = std::make_pair(MaxDiff, PhiIsSwapped);
3932 for (
int cycle = getFinalCycle() + 1; cycle <= LastCycle; ++cycle)
3933 ScheduledInstrs.erase(cycle);
3937 for (
int i = 0, e = SSD->SUnits.size(); i != e; ++
i) {
3938 SUnit *SU = &SSD->SUnits[
i];
3939 SSD->applyInstrChange(SU->
getInstr(), *
this,
true);
3944 for (
int Cycle = getFirstCycle(), E = getFinalCycle(); Cycle <=
E; ++Cycle) {
3945 std::deque<SUnit *> &cycleInstrs = ScheduledInstrs[Cycle];
3946 std::deque<SUnit *> newOrderZC;
3948 for (
unsigned i = 0, e = cycleInstrs.size(); i < e; ++
i) {
3949 SUnit *SU = cycleInstrs[
i];
3951 orderDependence(SSD, SU, newOrderZC);
3953 std::deque<SUnit *> newOrderI;
3955 for (
unsigned i = 0, e = cycleInstrs.size(); i < e; ++
i) {
3956 SUnit *SU = cycleInstrs[
i];
3958 orderDependence(SSD, SU, newOrderI);
3961 cycleInstrs.swap(newOrderZC);
3962 cycleInstrs.insert(cycleInstrs.end(), newOrderI.begin(), newOrderI.end());
3971 for (
int cycle = getFirstCycle(); cycle <= getFinalCycle(); ++cycle) {
3973 const_sched_iterator cycleInstrs = ScheduledInstrs.find(cycle);
3974 for (
SUnit *CI : cycleInstrs->second) {
3975 os <<
"cycle " << cycle <<
" (" << stageScheduled(CI) <<
") ";
3976 os <<
"(" << CI->
NodeNum <<
") ";
bool canReserveResources(const llvm::MCInstrDesc *MID)
Pass interface - Implemented by all 'passes'.
void print(raw_ostream &OS, bool SkipOpers=false, const TargetInstrInfo *TII=nullptr) const
void push_back(const T &Elt)
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
std::vector< int >::const_reverse_iterator const_reverse_iterator
A parsed version of the target data layout string in and methods for querying it. ...
const_iterator end(StringRef path)
Get end iterator over path.
static cl::opt< bool > SwpIgnoreRecMII("pipeliner-ignore-recmii", cl::ReallyHidden, cl::init(false), cl::ZeroOrMore, cl::desc("Ignore RecMII"))
virtual void finishBlock()
finishBlock - Clean up after scheduling in the given block.
mop_iterator operands_end()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
instr_iterator instr_begin()
static cl::opt< bool > SwpPruneDeps("pipeliner-prune-deps", cl::desc("Prune dependences between unrelated Phi nodes."), cl::Hidden, cl::init(true))
A command line option to disable the pruning of chain dependences due to an unrelated Phi...
static void computeLiveOuts(MachineFunction &MF, RegPressureTracker &RPTracker, NodeSet &NS)
Compute the live-out registers for the instructions in a node-set.
instr_iterator instr_end()
STATISTIC(NumFunctions,"Total number of functions")
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
MachineBasicBlock * getMBB() const
auto remove_if(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range))
Provide wrappers to std::remove_if which take ranges instead of having to pass begin/end explicitly...
aarch64 AArch64 CCMP Pass
unsigned createVirtualRegister(const TargetRegisterClass *RegClass)
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
DFAPacketizer * CreateTargetScheduleState(const TargetSubtargetInfo &STI) const override
Create machine specific model for scheduling.
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
MachineInstr * getInstr() const
getInstr - Return the representative MachineInstr for this SUnit.
bool operator>(int64_t V1, const APSInt &V2)
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
void transferSuccessors(MachineBasicBlock *FromMBB)
Transfers all the successors from MBB to this machine basic block (i.e., copies all the successors Fr...
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
static cl::opt< bool > EnableSWPOptSize("enable-pipeliner-opt-size", cl::desc("Enable SWP at Os."), cl::Hidden, cl::init(false))
A command line option to enable SWP at -Os.
static void getPhiRegs(MachineInstr &Phi, MachineBasicBlock *Loop, unsigned &InitVal, unsigned &LoopVal)
Return the register values for the operands of a Phi instruction.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
const_iterator begin(StringRef path)
Get begin iterator over path.
This class implements a map that also provides access to all stored values in a deterministic order...
static void dump(StringRef Title, SpillInfo const &Spills)
The two locations do not alias at all.
static bool computePath(SUnit *Cur, SetVector< SUnit * > &Path, SetVector< SUnit * > &DestNodes, SetVector< SUnit * > &Exclude, SmallPtrSet< SUnit *, 8 > &Visited)
Return true if there is a path from the specified node to any of the nodes in DestNodes.
void removePred(const SDep &D)
removePred - This removes the specified edge as a pred of the current node if it exists.
void GetUnderlyingObjects(Value *V, SmallVectorImpl< Value * > &Objects, const DataLayout &DL, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to GetUnderlyingObject except that it can look through phi and select instruct...
vector_type::const_iterator const_iterator
iterator_range< mmo_iterator > memoperands()
iterator_range< mop_iterator > operands()
iterator end()
Get an iterator to the end of the SetVector.
const std::vector< BlockT * > & getBlocks() const
Get a list of the basic blocks which make up this loop.
size_type size() const
Determine the number of elements in the SetVector.
bool isArtificial() const
isArtificial - Test if this is an Order dependence that is marked as "artificial", meaning it isn't necessary for correctness.
BlockT * getHeader() const
SmallVector< SDep, 4 > Preds
A register anti-dependedence (aka WAR).
bool isDereferenceableInvariantLoad(AliasAnalysis *AA) const
Return true if this load instruction never traps and points to a memory location whose value doesn't ...
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
Analyze the branching code at the end of MBB, returning true if it cannot be understood (e...
AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB)
The main low level interface to the alias analysis implementation.
unsigned getHeight() const
getHeight - Return the height of this node, which is the length of the maximum path down to any node ...
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
A description of a memory reference used in the backend.
static use_iterator use_end()
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static bool isDependenceBarrier(MachineInstr &MI, AliasAnalysis *AA)
Return true if the instruction causes a chain between memory references before and after it...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
Kind getKind() const
getKind - Return an enum value representing the kind of the dependence.
void setInstr(MachineInstr *MI)
setInstr - Assign the instruction for the SUnit.
static cl::opt< int > SwpMaxMii("pipeliner-max-mii", cl::desc("Size limit for the the MII."), cl::Hidden, cl::init(27))
A command line argument to limit minimum initial interval for pipelining.
Modulo Software Pipelining
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
bool isReg() const
isReg - Tests if this is a MO_Register operand.
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Regular data dependence (aka true-dependence).
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const TargetRegisterClass * getRegClass(unsigned Reg) const
Return the register class of the specified virtual register.
std::vector< MachineBasicBlock * >::iterator succ_iterator
Reg
All possible values of the reg field in the ModR/M byte.
This file contains the simple types necessary to represent the attributes associated with functions a...
LLVM_NODISCARD bool empty() const
unsigned getNumOperands() const
Access to explicit operands of the instruction.
static bool ignoreDependence(const SDep &D, bool isPred)
Return true for DAG nodes that we ignore when computing the cost functions.
defusechain_iterator - This class provides iterator support for machine operands in the function that...
void RemoveOperand(unsigned i)
Erase an operand from an instruction, leaving it with one fewer operand than it started with...
bool insert(const value_type &X)
Insert a new element into the SetVector.
virtual bool getBaseAndOffsetPosition(const MachineInstr &MI, unsigned &BasePos, unsigned &OffsetPos) const
Return true if the instruction contains a base register and offset.
static void swapAntiDependences(std::vector< SUnit > &SUnits)
Swap all the anti dependences in the DAG.
static bool isSubset(S1Ty &Set1, S2Ty &Set2)
Return true if Set1 is a subset of Set2.
iterator begin()
Get an iterator to the beginning of the SetVector.
bool isPred(SUnit *N)
isPred - Test if node N is a predecessor of this node.
bool empty() const
Determine if the SetVector is empty or not.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
virtual void adjustSchedDependency(SUnit *def, SUnit *use, SDep &dep) const
Itinerary data supplied by a subtarget to be used by a target.
static cl::opt< bool > EnableSWP("enable-pipeliner", cl::Hidden, cl::init(true), cl::ZeroOrMore, cl::desc("Enable Software Pipelining"))
A command line option to turn software pipelining on or off.
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(std::begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
void reserveResources(const llvm::MCInstrDesc *MID)
iterator find(const KeyT &Key)
static int getLatency(LLVMDisasmContext *DC, const MCInst &Inst)
Gets latency information for Inst, based on DC information.
const TargetRegisterClass * constrainRegClass(unsigned Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
reverse_iterator rbegin()
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
static void getUnderlyingObjects(MachineInstr *MI, SmallVectorImpl< Value * > &Objs, const DataLayout &DL)
Return the underlying objects for the memory references of an instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
const MachineBasicBlock * getParent() const
TargetInstrInfo - Interface to description of machine instruction set.
static void removePhis(MachineBasicBlock *BB, MachineBasicBlock *Incoming)
Remove the incoming block from the Phis in a basic block.
mmo_iterator memoperands_end() const
SDep - Scheduling dependency.
static unsigned getInitPhiReg(MachineInstr &Phi, MachineBasicBlock *LoopBB)
Return the Phi register value that comes from the incoming block.
void addLiveRegs(ArrayRef< RegisterMaskPair > Regs)
Force liveness of virtual registers or physical register units.
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
initializer< Ty > init(const Ty &Val)
friend const_iterator end(StringRef path)
Get end iterator over path.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned const MachineRegisterInfo * MRI
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
RegisterPressure computed within a region of instructions delimited by TopIdx and BottomIdx...
static bool hasUseAfterLoop(unsigned Reg, MachineBasicBlock *BB, MachineRegisterInfo &MRI)
Return true if the register has a use that occurs outside the specified loop.
bool getIncrementValue(const MachineInstr &MI, int &Value) const override
If the instruction is an increment of a constant value, return the amount.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
static bool pred_L(SetVector< SUnit * > &NodeOrder, SmallSetVector< SUnit *, 8 > &Preds, const NodeSet *S=nullptr)
Compute the Pred_L(O) set, as defined in the paper.
MachineInstrBuilder & UseMI
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
static int64_t computeDelta(SectionEntry *A, SectionEntry *B)
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
const MachineOperand & getOperand(unsigned i) const
bool getBaseAndOffsetPosition(const MachineInstr &MI, unsigned &BasePos, unsigned &OffsetPos) const override
For instructions with a base and offset, return the position of the base register and offset operands...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool isRegTiedToUseOperand(unsigned DefOpIdx, unsigned *UseOpIdx=nullptr) const
Given the index of a register def operand, check if the register def is tied to a source operand...
static bool isIntersect(SmallSetVector< SUnit *, 8 > &Set1, const NodeSet &Set2, SmallSetVector< SUnit *, 8 > &Result)
Return true if Set1 contains elements in Set2.
void setMBB(MachineBasicBlock *MBB)
AliasResult
The possible results of an alias query.
Represent the analysis usage information of a pass.
unsigned getLatency() const
getLatency - Return the latency value for this edge, which roughly means the minimum number of cycles...
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
static const unsigned End
Track the current register pressure at some position in the instruction stream, and remember the high...
void setImm(int64_t immVal)
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore...
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
LLVM_NODISCARD bool empty() const
succ_iterator succ_begin()
bool isZeroCost(unsigned Opcode) const
Return true for pseudo instructions that don't consume any machine resources in their current form...
INITIALIZE_PASS_BEGIN(MachinePipeliner,"pipeliner","Modulo Software Pipelining", false, false) INITIALIZE_PASS_END(MachinePipeliner
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Any other ordering dependency.
static cl::opt< unsigned > MaxIter("bb-vectorize-max-iter", cl::init(0), cl::Hidden, cl::desc("The maximum number of pairing iterations"))
bool isPostIncrement(const MachineInstr &MI) const override
Return true for post-incremented instructions.
static void replaceRegUsesAfterLoop(unsigned FromReg, unsigned ToReg, MachineBasicBlock *MBB, MachineRegisterInfo &MRI, LiveIntervals &LIS)
Replace all uses of FromReg that appear outside the specified basic block with ToReg.
An unknown scheduling barrier.
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
Representation for a specific memory location.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
std::vector< unsigned > MaxSetPressure
Map of max reg pressure indexed by pressure set ID, not class ID.
unsigned reduceLoopCount(MachineBasicBlock &MBB, MachineInstr *IndVar, MachineInstr &Cmp, SmallVectorImpl< MachineOperand > &Cond, SmallVectorImpl< MachineInstr * > &PrevInsts, unsigned Iter, unsigned MaxIter) const override
Generate code to reduce the loop iteration by one and check if the loop is finished.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
A SetVector that performs no allocations if smaller than a certain size.
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
unsigned countPopulation(T Value)
Count the number of set bits in a value.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
reverse_instr_iterator instr_rbegin()
static cl::opt< int > SwpLoopLimit("pipeliner-max", cl::Hidden, cl::init(-1))
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false...
friend const_iterator begin(StringRef path)
Get begin iterator over path.
static cl::opt< int > SwpMaxStages("pipeliner-max-stages", cl::desc("Maximum stages allowed in the generated scheduled."), cl::Hidden, cl::init(3))
A command line argument to limit the number of stages in the pipeline.
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
static LaneBitmask getNone()
static bool succ_L(SetVector< SUnit * > &NodeOrder, SmallSetVector< SUnit *, 8 > &Succs, const NodeSet *S=nullptr)
Compute the Succ_L(O) set, as defined in the paper.
char & MachinePipelinerID
This pass performs software pipelining on machine instructions.
LLVM_NODISCARD T pop_back_val()
bool isAllocatable(unsigned PhysReg) const
isAllocatable - Returns true when PhysReg belongs to an allocatable register class and it hasn't been...
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &DL, int *BytesAdded=nullptr) const override
Insert branch code into the end of the specified MachineBasicBlock.
void dump(const TargetInstrInfo *TII=nullptr) const
reverse_iterator rbegin()
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static bool isSuccOrder(SUnit *SUa, SUnit *SUb)
Return true if SUb can be reached from SUa following the chain edges.
reverse_instr_iterator instr_rend()
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
std::vector< int >::const_iterator const_iterator
void clear()
Completely clear the SetVector.
size_type count(const KeyT &Key) const
const Value * getValue() const
Return the base address of the memory access.
static bool hasDataDependence(SUnit *Inst1, SUnit *Inst2)
Return true if Inst1 defines a value that is used in Inst2.
std::set< NodeId > NodeSet
static SUnit * multipleIterations(SUnit *SU, SwingSchedulerDAG *DAG)
If an instruction has a use that spans multiple iterations, then return true.
void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New)
Replace successor OLD with NEW and update probability info.
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
Remove the branching code at the end of the specific MBB.
LiveInterval & createEmptyInterval(unsigned Reg)
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
static void clear(coro::Shape &Shape)
bool isBoundaryNode() const
Boundary nodes are placeholders for the boundary of the scheduling region.
unsigned getDepth() const
getDepth - Return the depth of this node, which is the length of the maximum path up to any node whic...
void setLatency(unsigned Lat)
setLatency - Set the latency for this edge.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
void initializeMachinePipelinerPass(PassRegistry &)
TargetSubtargetInfo - Generic base class for all target subtargets.
bool operator!=(uint64_t V1, const APInt &V2)
ScheduleDAGInstrs - A ScheduleDAG subclass for scheduling lists of MachineInstrs. ...
Representation of each machine instruction.
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
unsigned getSchedClass() const
Return the scheduling class for this instruction.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
unsigned getNumBlocks() const
Get the number of blocks in this loop in constant time.
These values represent a non-pipelined step in the execution of an instruction.
Represents a single loop in the control flow graph.
use_iterator use_begin(unsigned RegNo) const
This file provides utility analysis objects describing memory locations.
void setReg(unsigned Reg)
Change the register this operand corresponds to.
void push_back(MachineInstr *MI)
bool isCall(QueryType Type=AnyInBundle) const
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
iterator find(const KeyT &Val)
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
bool hasInterval(unsigned Reg) const
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
MachineInstr * getVRegDef(unsigned Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
unsigned getReg() const
getReg - Returns the register number.
Store the effects of a change in pressure on things that MI scheduler cares about.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool addPred(const SDep &D, bool Required=true)
addPred - This adds the specified edge as a pred of the current node if not already.
virtual const TargetInstrInfo * getInstrInfo() const
mop_iterator operands_begin()
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
vector_type::const_iterator iterator
A vector that has set insertion semantics.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
SmallVector< SDep, 4 > Succs
This class implements an extremely fast bulk output stream that can only output to a stream...
static cl::opt< bool > SwpPruneLoopCarried("pipeliner-prune-loop-carried", cl::desc("Prune loop carried order dependences."), cl::Hidden, cl::init(true))
A command line option to disable the pruning of loop carried order dependences.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
bool operator==(uint64_t V1, const APInt &V2)
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
bool getMemOpBaseRegImmOfs(MachineInstr &LdSt, unsigned &BaseReg, int64_t &Offset, const TargetRegisterInfo *TRI) const override
Get the base register and byte offset of a load/store instr.
PriorityQueue - This class behaves like std::priority_queue and provides a few additional convenience...
unsigned getReg() const
getReg - Return the register associated with this edge.
ScheduleDAGTopologicalSort is a class that computes a topological ordering for SUnits and provides me...
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
bool analyzeLoop(MachineLoop &L, MachineInstr *&IndVarInst, MachineInstr *&CmpInst) const override
Analyze the loop code, return true if it cannot be understood.
static unsigned getLoopPhiReg(MachineInstr &Phi, MachineBasicBlock *LoopBB)
Return the Phi register value that comes the the loop block.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd)
Assign this MachineInstr's memory reference descriptor list.
bool areMemAccessesTriviallyDisjoint(MachineInstr &MIa, MachineInstr &MIb, AliasAnalysis *AA=nullptr) const override
void dump(const ScheduleDAG *G) const
SUnit - Scheduling unit.
SUnit - Scheduling unit. This is a node in the scheduling DAG.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
void tieOperands(unsigned DefIdx, unsigned UseIdx)
Add a tie between the register operands at DefIdx and UseIdx.