30#define DEBUG_TYPE "igrouplp"
36 cl::desc(
"Whether to use the exponential time solver to fit "
37 "the instructions to the pipeline as closely as "
43 cl::desc(
"The maximum number of scheduling group conflicts "
44 "which we attempt to solve with the exponential time "
45 "exact solver. Problem sizes greater than this will"
46 "be solved by the less accurate greedy algorithm. Selecting "
47 "solver by size is superseded by manually selecting "
48 "the solver (e.g. by amdgpu-igrouplp-exact-solver"));
52 cl::desc(
"The amount of branches that we are willing to explore with"
53 "the exact algorithm before giving up."));
57 cl::desc(
"Whether to use the cost heuristic to make choices as we "
58 "traverse the search space using the exact solver. Defaulted "
59 "to on, and if turned off, we will use the node order -- "
60 "attempting to put the later nodes in the later sched groups. "
61 "Experimentally, results are mixed, so this should be set on a "
62 "case-by-case basis."));
66enum class SchedGroupMask {
79 ALL = ALU | VALU | SALU | MFMA | VMEM | VMEM_READ | VMEM_WRITE | DS |
80 DS_READ | DS_WRITE | TRANS,
89class InstructionRule {
95 std::optional<SmallVector<SUnit *, 4>> Cache;
105 bool NeedsCache =
false)
112 virtual ~InstructionRule() =
default;
125 SchedGroupMask SGMask;
128 std::optional<unsigned> MaxSize;
141 static unsigned NumSchedGroups;
158 bool canAddSU(
SUnit &SU)
const;
163 void link(
SUnit &SU,
bool MakePred =
false);
167 int link(
SUnit &SU,
bool MakePred,
168 std::vector<std::pair<SUnit *, SUnit *>> &AddedEdges);
177 void link(SchedGroup &OtherGroup);
180 bool isFull()
const {
return MaxSize && Collection.
size() >= *MaxSize; }
186 void addRule(std::shared_ptr<InstructionRule> NewRule) {
191 bool allowedByRules(
const SUnit *SU,
195 for (
size_t I = 0;
I < Rules.
size();
I++) {
196 auto TheRule = Rules[
I].get();
197 if (!TheRule->apply(SU, Collection, SyncPipe)) {
205 void add(
SUnit &SU) {
207 <<
format_hex((
int)SGMask, 10,
true) <<
" adding "
213 void pop() { Collection.
pop_back(); }
216 void initSchedGroup();
223 void initSchedGroup(std::vector<SUnit>::reverse_iterator RIter,
224 SUnitsToCandidateSGsMap &SyncedInstrs);
226 void initSchedGroup(SUnitsToCandidateSGsMap &SyncedInstrs);
228 int getSyncID() {
return SyncID; }
230 int getSGID() {
return SGID; }
232 SchedGroupMask getMask() {
return SGMask; }
234 SchedGroup(SchedGroupMask SGMask, std::optional<unsigned> MaxSize,
236 : SGMask(SGMask), MaxSize(MaxSize), DAG(DAG),
TII(
TII) {
237 SGID = NumSchedGroups++;
240 SchedGroup(SchedGroupMask SGMask, std::optional<unsigned> MaxSize,
int SyncID,
242 : SGMask(SGMask), MaxSize(MaxSize), SyncID(SyncID), DAG(DAG),
TII(
TII) {
243 SGID = NumSchedGroups++;
253 while (!SU.
Preds.empty())
257 while (!SU.
Succs.empty())
258 for (
auto &S : SU.
Succs)
259 for (
auto &SP : S.getSUnit()->Preds)
260 if (SP.getSUnit() == &SU)
261 S.getSUnit()->removePred(SP);
264typedef std::pair<SUnit *, SmallVector<int, 4>> SUToCandSGsPair;
276class PipelineSolver {
289 bool NeedsSolver =
false;
293 unsigned computeProblemSize();
304 int CurrConflInstNo = 0;
306 int CurrSyncGroupIdx = 0;
308 int BeginSyncGroupIdx = 0;
317 void advancePosition();
320 void retreatPosition();
329 template <
typename T>
330 void greedyFind(std::vector<std::pair<SUnit *, SUnit *>> &AddedEdges,
T I,
336 template <
typename T>
343 template <
typename T>
void linkSchedGroups(
T I,
T E);
347 std::vector<std::pair<SUnit *, SUnit *>> &AddedEdges);
351 template <
typename T>
352 int linkSUnit(
SUnit *SU,
int SGID,
353 std::vector<std::pair<SUnit *, SUnit *>> &AddedEdges,
T I,
T E);
355 void removeEdges(
const std::vector<std::pair<SUnit *, SUnit *>> &AddedEdges);
357 void convertSyncMapsToArrays();
369 : DAG(DAG), SyncedInstrs(SyncedInstrs),
370 SyncedSchedGroups(SyncedSchedGroups), IsBottomUp(IsBottomUp) {
372 for (
auto &PipelineInstrs : SyncedInstrs) {
373 if (PipelineInstrs.second.
size() > 0) {
382 convertSyncMapsToArrays();
384 CurrPipeline = BestPipeline;
386 while (
static_cast<size_t>(BeginSyncGroupIdx) < PipelineInstrs.
size() &&
387 PipelineInstrs[BeginSyncGroupIdx].
size() == 0)
390 if (
static_cast<size_t>(BeginSyncGroupIdx) >= PipelineInstrs.
size())
395void PipelineSolver::reset() {
397 for (
auto &SyncPipeline : CurrPipeline) {
398 for (
auto &SG : SyncPipeline) {
400 SG.Collection.
clear();
404 if (SchedBarr != TempCollection.
end())
405 SG.Collection.push_back(*SchedBarr);
409 CurrSyncGroupIdx = BeginSyncGroupIdx;
414void PipelineSolver::convertSyncMapsToArrays() {
415 for (
auto &SyncPipe : SyncedSchedGroups) {
416 BestPipeline.insert(BestPipeline.begin(), SyncPipe.second);
419 int PipelineIDx = SyncedInstrs.size() - 1;
420 PipelineInstrs.resize(SyncedInstrs.size());
421 for (
auto &SyncInstrMap : SyncedInstrs) {
422 for (
auto &SUsToCandSGs : SyncInstrMap.second) {
423 if (PipelineInstrs[PipelineIDx].
size() == 0) {
424 PipelineInstrs[PipelineIDx].push_back(
425 std::pair(SUsToCandSGs.first, SUsToCandSGs.second));
428 auto SortPosition = PipelineInstrs[PipelineIDx].begin();
431 while (SortPosition != PipelineInstrs[PipelineIDx].end() &&
432 SUsToCandSGs.first->NodeNum > SortPosition->first->NodeNum)
434 PipelineInstrs[PipelineIDx].insert(
435 SortPosition, std::pair(SUsToCandSGs.first, SUsToCandSGs.second));
441template <
typename T>
void PipelineSolver::linkSchedGroups(
T I,
T E) {
442 for (;
I != E; ++
I) {
444 for (
auto J = std::next(
I); J != E; ++J) {
451void PipelineSolver::makePipeline() {
453 for (
auto &SyncPipeline : BestPipeline) {
455 for (
auto &SG : SyncPipeline) {
458 SUnit *SGBarr =
nullptr;
459 for (
auto &SU : SG.Collection) {
467 resetEdges(*SGBarr, DAG);
468 SG.link(*SGBarr,
false);
472 for (
auto &SyncPipeline : BestPipeline) {
473 IsBottomUp ? linkSchedGroups(SyncPipeline.rbegin(), SyncPipeline.rend())
474 : linkSchedGroups(SyncPipeline.begin(), SyncPipeline.end());
479int PipelineSolver::linkSUnit(
480 SUnit *SU,
int SGID, std::vector<std::pair<SUnit *, SUnit *>> &AddedEdges,
482 bool MakePred =
false;
485 if (
I->getSGID() == SGID) {
490 AddedCost += Group.link(*SU, MakePred, AddedEdges);
496int PipelineSolver::addEdges(
498 std::vector<std::pair<SUnit *, SUnit *>> &AddedEdges) {
508 return IsBottomUp ? linkSUnit(SU, SGID, AddedEdges, SyncPipeline.
rbegin(),
510 : linkSUnit(SU, SGID, AddedEdges, SyncPipeline.
begin(),
514void PipelineSolver::removeEdges(
515 const std::vector<std::pair<SUnit *, SUnit *>> &EdgesToRemove) {
518 for (
auto &PredSuccPair : EdgesToRemove) {
519 SUnit *Pred = PredSuccPair.first;
520 SUnit *Succ = PredSuccPair.second;
523 Succ->
Preds, [&Pred](
SDep &
P) { return P.getSUnit() == Pred; });
531void PipelineSolver::advancePosition() {
534 if (
static_cast<size_t>(CurrConflInstNo) >=
535 PipelineInstrs[CurrSyncGroupIdx].
size()) {
539 while (
static_cast<size_t>(CurrSyncGroupIdx) < PipelineInstrs.size() &&
540 PipelineInstrs[CurrSyncGroupIdx].size() == 0)
545void PipelineSolver::retreatPosition() {
546 assert(CurrConflInstNo >= 0);
547 assert(CurrSyncGroupIdx >= 0);
549 if (CurrConflInstNo > 0) {
554 if (CurrConflInstNo == 0) {
557 if (CurrSyncGroupIdx == BeginSyncGroupIdx)
562 while (PipelineInstrs[CurrSyncGroupIdx].
size() == 0)
565 CurrConflInstNo = PipelineInstrs[CurrSyncGroupIdx].size() - 1;
569bool PipelineSolver::checkOptimal() {
570 if (
static_cast<size_t>(CurrSyncGroupIdx) == PipelineInstrs.size()) {
571 if (BestCost == -1 || CurrCost < BestCost) {
572 BestPipeline = CurrPipeline;
579 bool DoneExploring =
false;
580 if (MaxBranchesExplored > 0 && BranchesExplored >= MaxBranchesExplored)
581 DoneExploring =
true;
583 return (DoneExploring || BestCost == 0);
587void PipelineSolver::populateReadyList(
589 SUToCandSGsPair CurrSU = PipelineInstrs[CurrSyncGroupIdx][CurrConflInstNo];
590 auto SyncPipeline = CurrPipeline[CurrSyncGroupIdx];
591 assert(CurrSU.second.size() >= 1);
593 for (;
I != E; ++
I) {
594 std::vector<std::pair<SUnit *, SUnit *>> AddedEdges;
597 return SG.getSGID() == CandSGID;
602 if (
Match->isFull()) {
603 ReadyList.push_back(std::pair(*
I, MissPenalty));
607 int TempCost = addEdges(SyncPipeline, CurrSU.first, CandSGID, AddedEdges);
608 ReadyList.push_back(std::pair(*
I, TempCost));
609 removeEdges(AddedEdges);
611 ReadyList.push_back(std::pair(*
I, -1));
615 std::sort(ReadyList.begin(), ReadyList.end(),
616 [](std::pair<int, int>
A, std::pair<int, int>
B) {
617 return A.second < B.second;
621 assert(ReadyList.size() == CurrSU.second.size());
624bool PipelineSolver::solveExact() {
628 if (
static_cast<size_t>(CurrSyncGroupIdx) == PipelineInstrs.size())
631 assert(
static_cast<size_t>(CurrSyncGroupIdx) < PipelineInstrs.size());
632 assert(
static_cast<size_t>(CurrConflInstNo) <
633 PipelineInstrs[CurrSyncGroupIdx].
size());
634 SUToCandSGsPair CurrSU = PipelineInstrs[CurrSyncGroupIdx][CurrConflInstNo];
636 <<
") in Pipeline # " << CurrSyncGroupIdx <<
"\n");
641 IsBottomUp ? populateReadyList(ReadyList, CurrSU.second.
rbegin(),
642 CurrSU.second.rend())
643 : populateReadyList(ReadyList, CurrSU.second.
begin(),
644 CurrSU.second.end());
646 auto I = ReadyList.
begin();
647 auto E = ReadyList.
end();
648 for (;
I != E; ++
I) {
652 if (BestCost != -1 && (CurrCost +
I->second > BestCost))
655 int CandSGID =
I->first;
657 std::vector<std::pair<SUnit *, SUnit *>> AddedEdges;
658 auto &SyncPipeline = CurrPipeline[CurrSyncGroupIdx];
660 for (
auto &SG : SyncPipeline) {
661 if (SG.getSGID() == CandSGID)
668 if (!
Match->allowedByRules(CurrSU.first, SyncPipeline))
672 << (
int)
Match->getMask() <<
"and ID " << CandSGID
674 Match->add(*CurrSU.first);
675 AddedCost = addEdges(SyncPipeline, CurrSU.first, CandSGID, AddedEdges);
676 LLVM_DEBUG(
dbgs() <<
"Cost of Assignment: " << AddedCost <<
"\n");
677 CurrCost += AddedCost;
680 bool FinishedExploring =
false;
683 if (CurrCost < BestCost || BestCost == -1) {
685 FinishedExploring = BestCost != 0;
686 if (!FinishedExploring)
692 CurrCost -= AddedCost;
693 removeEdges(AddedEdges);
695 CurrPipeline[CurrSyncGroupIdx] = SyncPipeline;
696 if (FinishedExploring)
703 CurrCost += MissPenalty;
706 LLVM_DEBUG(
dbgs() <<
"NOT Assigned (" << CurrSU.first->NodeNum <<
")\n");
708 bool FinishedExploring =
false;
709 if (CurrCost < BestCost || BestCost == -1) {
711 bool FinishedExploring = BestCost != 0;
712 if (!FinishedExploring)
718 CurrCost -= MissPenalty;
719 return FinishedExploring;
723void PipelineSolver::greedyFind(
724 std::vector<std::pair<SUnit *, SUnit *>> &AddedEdges,
T I,
T E) {
725 SUToCandSGsPair CurrSU = PipelineInstrs[CurrSyncGroupIdx][CurrConflInstNo];
726 int BestNodeCost = -1;
728 SchedGroup *BestGroup =
nullptr;
729 int BestGroupID = -1;
730 auto &SyncPipeline = CurrPipeline[CurrSyncGroupIdx];
732 <<
") in Pipeline # " << CurrSyncGroupIdx <<
"\n");
738 for (;
I != E; ++
I) {
739 std::vector<std::pair<SUnit *, SUnit *>> AddedEdges;
742 return SG.getSGID() == CandSGID;
746 LLVM_DEBUG(
dbgs() <<
"Trying SGID # " << CandSGID <<
" with Mask "
747 << (
int)
Match->getMask() <<
"\n");
749 if (
Match->isFull()) {
753 if (!
Match->allowedByRules(CurrSU.first, SyncPipeline)) {
754 LLVM_DEBUG(
dbgs() <<
"SGID # " << CandSGID <<
" has conflicting rule\n");
757 TempCost = addEdges(SyncPipeline, CurrSU.first, CandSGID, AddedEdges);
759 if (TempCost < BestNodeCost || BestNodeCost == -1) {
761 BestNodeCost = TempCost;
762 BestGroupID = CandSGID;
764 removeEdges(AddedEdges);
765 if (BestNodeCost == 0)
769 if (BestGroupID != -1) {
770 BestGroup->add(*CurrSU.first);
771 addEdges(SyncPipeline, CurrSU.first, BestGroupID, AddedEdges);
772 LLVM_DEBUG(
dbgs() <<
"Best Group has ID: " << BestGroupID <<
" and Mask"
773 << (
int)BestGroup->getMask() <<
"\n");
774 BestCost += TempCost;
776 BestCost += MissPenalty;
778 CurrPipeline[CurrSyncGroupIdx] = SyncPipeline;
781bool PipelineSolver::solveGreedy() {
783 std::vector<std::pair<SUnit *, SUnit *>> AddedEdges;
785 while (
static_cast<size_t>(CurrSyncGroupIdx) < PipelineInstrs.size()) {
786 SUToCandSGsPair CurrSU = PipelineInstrs[CurrSyncGroupIdx][CurrConflInstNo];
788 ? greedyFind(AddedEdges, CurrSU.second.rbegin(), CurrSU.second.rend())
789 : greedyFind(AddedEdges, CurrSU.second.begin(), CurrSU.second.end());
792 BestPipeline = CurrPipeline;
793 removeEdges(AddedEdges);
797unsigned PipelineSolver::computeProblemSize() {
798 unsigned ProblemSize = 0;
799 for (
auto &PipeConflicts : PipelineInstrs) {
800 ProblemSize += PipeConflicts.size();
806void PipelineSolver::solve() {
810 unsigned ProblemSize = computeProblemSize();
813 bool BelowCutoff = (CutoffForExact > 0) && ProblemSize <= CutoffForExact;
814 MissPenalty = (ProblemSize / 2) + 1;
817 if (EnableExactSolver || BelowCutoff) {
821 LLVM_DEBUG(
dbgs() <<
"Greedy produced best cost of " << BestCost <<
"\n");
825 LLVM_DEBUG(
dbgs() <<
"Exact produced best cost of " << BestCost <<
"\n");
837enum IGLPStrategyID :
int {
838 MFMASmallGemmOptID = 0,
839 MFMASmallGemmSingleWaveOptID = 1,
840 MFMAExpInterleave = 2
852 virtual bool applyIGLPStrategy(
866 virtual ~IGLPStrategy() =
default;
869class MFMASmallGemmOpt final :
public IGLPStrategy {
872 bool applyIGLPStrategy(
883 : IGLPStrategy(DAG,
TII) {
888bool MFMASmallGemmOpt::applyIGLPStrategy(
893 unsigned MFMACount = 0;
895 if (
TII->isMFMAorWMMA(
I))
898 const unsigned PipelineSyncID = 0;
899 SchedGroup *SG =
nullptr;
900 for (
unsigned I = 0;
I < MFMACount * 3; ++
I) {
901 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
902 SchedGroupMask::DS, 2, PipelineSyncID, DAG,
TII);
903 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
905 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
906 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG,
TII);
907 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
913class MFMAExpInterleaveOpt final :
public IGLPStrategy {
916 static unsigned TransPipeCount;
918 static unsigned MFMAPipeCount;
920 static unsigned AddPipeCount;
922 static unsigned MFMAEnablement;
924 static unsigned ExpRequirement;
926 static unsigned MFMAChains;
928 static unsigned MFMAChainLength;
933 static bool HasChainBetweenCvt;
935 static std::optional<unsigned> FirstPipeDSR;
944 class IsPipeExp final :
public InstructionRule {
949 auto DAG = SyncPipe[0].DAG;
951 if (Cache->empty()) {
953 auto E = DAG->
SUnits.rend();
954 for (;
I != E;
I++) {
955 if (
TII->isMFMAorWMMA(*
I->getInstr()))
956 Cache->push_back(&*
I);
962 auto Reaches = (std::any_of(
963 Cache->begin(), Cache->end(), [&SU, &DAG](
SUnit *TargetSU) {
964 return DAG->IsReachable(TargetSU, const_cast<SUnit *>(SU));
969 IsPipeExp(
const SIInstrInfo *
TII,
unsigned SGID,
bool NeedsCache =
false)
970 : InstructionRule(
TII, SGID, NeedsCache) {}
975 class EnablesNthMFMA final :
public InstructionRule {
982 bool FoundTrans =
false;
983 unsigned Counter = 1;
984 auto DAG = SyncPipe[0].DAG;
986 if (Cache->empty()) {
990 auto E = DAG->
SUnits.end();
991 for (;
I != E;
I++) {
992 if (FoundTrans &&
TII->isMFMAorWMMA(*
I->getInstr())) {
994 Cache->push_back(&*
I);
999 if (!FoundTrans &&
TII->isTRANS(
I->getInstr()->getOpcode()))
1010 bool NeedsCache =
false)
1016 class EnablesNthMFMAInChain final :
public InstructionRule {
1024 auto DAG = SyncPipe[0].DAG;
1026 if (!SU || !
TII->isMFMAorWMMA(*ChainSeed->
getInstr()))
1029 if (Cache->empty()) {
1030 auto TempSU = ChainSeed;
1035 for (
auto &Succ : TempSU->Succs) {
1036 if (
TII->isMFMAorWMMA(*Succ.getSUnit()->getInstr())) {
1037 TempSU = Succ.getSUnit();
1046 Cache->push_back(TempSU);
1055 EnablesNthMFMAInChain(
unsigned Number,
SUnit *ChainSeed,
1057 bool NeedsCache =
false)
1059 ChainSeed(ChainSeed) {}
1065 class LessThanNSuccs final :
public InstructionRule {
1068 bool HasIntermediary =
false;
1073 if (!SyncPipe.
size())
1076 auto SuccSize = std::count_if(
1078 [](
const SDep &Succ) { return Succ.getKind() == SDep::Data; });
1079 if (SuccSize >= Size)
1082 if (HasIntermediary) {
1083 for (
auto Succ : SU->
Succs) {
1084 auto SuccSize = std::count_if(
1086 [](
const SDep &SuccSucc) {
1087 return SuccSucc.getKind() == SDep::Data;
1089 if (SuccSize >= Size)
1096 LessThanNSuccs(
unsigned Size,
const SIInstrInfo *
TII,
unsigned SGID,
1097 bool HasIntermediary =
false,
bool NeedsCache =
false)
1098 : InstructionRule(
TII, SGID, NeedsCache), Size(Size),
1099 HasIntermediary(HasIntermediary) {}
1106 class GreaterThanOrEqualToNSuccs final :
public InstructionRule {
1109 bool HasIntermediary =
false;
1114 if (!SyncPipe.
size())
1117 auto SuccSize = std::count_if(
1119 [](
const SDep &Succ) { return Succ.getKind() == SDep::Data; });
1120 if (SuccSize >= Size)
1123 if (HasIntermediary) {
1124 for (
auto Succ : SU->
Succs) {
1125 auto SuccSize = std::count_if(
1127 [](
const SDep &SuccSucc) {
1128 return SuccSucc.getKind() == SDep::Data;
1130 if (SuccSize >= Size)
1137 GreaterThanOrEqualToNSuccs(
unsigned Size,
const SIInstrInfo *
TII,
1138 unsigned SGID,
bool HasIntermediary =
false,
1139 bool NeedsCache =
false)
1140 : InstructionRule(
TII, SGID, NeedsCache), Size(Size),
1141 HasIntermediary(HasIntermediary) {}
1145 class IsCvt final :
public InstructionRule {
1150 return Opc == AMDGPU::V_CVT_F16_F32_e32 ||
1151 Opc == AMDGPU::V_CVT_I32_F32_e32;
1153 IsCvt(
const SIInstrInfo *
TII,
unsigned SGID,
bool NeedsCache =
false)
1154 : InstructionRule(
TII, SGID, NeedsCache) {}
1158 class IsFMA final :
public InstructionRule {
1165 IsFMA(
const SIInstrInfo *
TII,
unsigned SGID,
bool NeedsCache =
false)
1166 : InstructionRule(
TII, SGID, NeedsCache) {}
1170 class IsPipeAdd final :
public InstructionRule {
1176 IsPipeAdd(
const SIInstrInfo *
TII,
unsigned SGID,
bool NeedsCache =
false)
1177 : InstructionRule(
TII, SGID, NeedsCache) {}
1182 class IsSuccOfPrevNthGroup final :
public InstructionRule {
1184 unsigned Distance = 1;
1189 SchedGroup *OtherGroup =
nullptr;
1190 if (!SyncPipe.
size())
1193 for (
auto &PipeSG : SyncPipe) {
1194 if ((
unsigned)PipeSG.getSGID() == SGID - Distance)
1195 OtherGroup = &PipeSG;
1200 if (!OtherGroup->Collection.size())
1203 for (
auto &OtherEle : OtherGroup->Collection) {
1204 for (
auto &Succ : OtherEle->Succs) {
1205 if (Succ.getSUnit() == SU && Succ.getKind() ==
SDep::Data)
1213 unsigned SGID,
bool NeedsCache =
false)
1214 : InstructionRule(
TII, SGID, NeedsCache), Distance(Distance) {}
1219 class IsReachableFromPrevNthGroup final :
public InstructionRule {
1221 unsigned Distance = 1;
1226 SchedGroup *OtherGroup =
nullptr;
1227 if (!SyncPipe.
size())
1230 for (
auto &PipeSG : SyncPipe) {
1231 if ((
unsigned)PipeSG.getSGID() == SGID - Distance)
1232 OtherGroup = &PipeSG;
1237 if (!OtherGroup->Collection.size())
1240 auto DAG = SyncPipe[0].DAG;
1242 for (
auto &OtherEle : OtherGroup->Collection)
1248 IsReachableFromPrevNthGroup(
unsigned Distance,
const SIInstrInfo *
TII,
1249 unsigned SGID,
bool NeedsCache =
false)
1250 : InstructionRule(
TII, SGID, NeedsCache), Distance(Distance) {}
1254 class OccursAtOrAfterNode final :
public InstructionRule {
1265 bool NeedsCache =
false)
1271 class IsExactMFMA final :
public InstructionRule {
1279 if (!SU || !
TII->isMFMAorWMMA(*ChainSeed->
getInstr()))
1282 if (Cache->empty()) {
1283 auto TempSU = ChainSeed;
1288 for (
auto &Succ : TempSU->Succs) {
1289 if (
TII->isMFMAorWMMA(*Succ.getSUnit()->getInstr())) {
1290 TempSU = Succ.getSUnit();
1299 Cache->push_back(TempSU);
1305 return (*Cache)[0] == SU;
1309 unsigned SGID,
bool NeedsCache =
false)
1311 ChainSeed(ChainSeed) {}
1317 class OccursAfterExp final :
public InstructionRule {
1323 auto DAG = SyncPipe[0].DAG;
1324 if (Cache->empty()) {
1325 for (
auto &SU : DAG->
SUnits)
1334 return SU->
NodeNum > (*Cache)[0]->NodeNum;
1338 bool NeedsCache =
false)
1339 : InstructionRule(
TII, SGID, NeedsCache) {}
1343 bool applyIGLPStrategy(
1352 : IGLPStrategy(DAG,
TII) {
1357unsigned MFMAExpInterleaveOpt::TransPipeCount = 0;
1358unsigned MFMAExpInterleaveOpt::MFMAPipeCount = 0;
1359unsigned MFMAExpInterleaveOpt::AddPipeCount = 0;
1360unsigned MFMAExpInterleaveOpt::MFMAEnablement = 0;
1361unsigned MFMAExpInterleaveOpt::ExpRequirement = 0;
1362unsigned MFMAExpInterleaveOpt::MFMAChains = 0;
1363unsigned MFMAExpInterleaveOpt::MFMAChainLength = 0;
1364bool MFMAExpInterleaveOpt::HasCvt =
false;
1365bool MFMAExpInterleaveOpt::HasChainBetweenCvt =
false;
1366std::optional<unsigned> MFMAExpInterleaveOpt::FirstPipeDSR = std::nullopt;
1375 auto isBitPack = [](
unsigned Opc) {
1376 return Opc == AMDGPU::V_PACK_B32_F16_e64 || Opc == AMDGPU::V_PERM_B32_e64;
1379 auto isCvt = [](
unsigned Opc) {
1380 return Opc == AMDGPU::V_CVT_F16_F32_e32 || Opc == AMDGPU::V_CVT_I32_F32_e32;
1383 auto isAdd = [](
unsigned Opc) {
return Opc == AMDGPU::V_ADD_F32_e32; };
1388 if (
TII->isTRANS(Opc)) {
1390 if (SU.
Succs.size() >= 7)
1392 for (
auto &Succ : SU.
Succs) {
1393 if (Succ.getSUnit()->Succs.size() >= 7)
1412 if (!(PackSUs.
size() && MFMAPipeCands.
size() && ExpPipeCands.
size()))
1417 std::optional<SUnit *> TempMFMA;
1418 std::optional<SUnit *> TempExp;
1420 for (
auto &PredSU : ExpPipeCands) {
1421 for (
auto &SuccSU : MFMAPipeCands) {
1434 if (!(TempExp && TempMFMA))
1437 HasChainBetweenCvt =
1438 std::find_if((*TempExp)->Succs.begin(), (*TempExp)->Succs.end(),
1439 [&isCvt](
SDep &Succ) {
1440 return isCvt(Succ.getSUnit()->getInstr()->getOpcode());
1441 }) == (*TempExp)->Succs.end();
1444 for (
auto &SuccSU : MFMAPipeCands) {
1445 if (MFMAPipeSUs.
size() &&
1446 std::find_if(MFMAPipeSUs.
begin(), MFMAPipeSUs.
end(),
1447 [&SuccSU](
SUnit *PotentialMatch) {
1448 return PotentialMatch->NodeNum == SuccSU->NodeNum;
1449 }) != MFMAPipeSUs.
end())
1452 for (
auto &PredSU : ExpPipeCands) {
1460 MFMAPipeCount = MFMAPipeSUs.
size();
1462 assert(TempExp && TempMFMA);
1463 assert(MFMAPipeCount > 0);
1465 std::optional<SUnit *> TempCvt;
1466 for (
auto &SuccSU : CvtSUs) {
1474 if (TempCvt.has_value()) {
1475 for (
auto &SuccSU : MFMAPipeSUs) {
1484 for (
auto &MFMAPipeSU : MFMAPipeSUs) {
1487 if (!std::any_of(MFMAPipeSU->Preds.begin(), MFMAPipeSU->Preds.end(),
1489 return TII->isMFMAorWMMA(*Succ.getSUnit()->getInstr());
1491 MFMAChainSeeds.push_back(MFMAPipeSU);
1499 for (
auto Pred : MFMAChainSeeds[0]->Preds) {
1500 if (
TII->isDS(Pred.getSUnit()->getInstr()->getOpcode()) &&
1501 Pred.getSUnit()->getInstr()->mayLoad())
1502 FirstPipeDSR = Pred.getSUnit()->NodeNum;
1505 MFMAChainLength = MFMAPipeCount / MFMAChains;
1508 unsigned PackSuccCount = std::count_if(
1509 PackSUs.
begin(), PackSUs.
end(), [
this, &TempExp](
SUnit *VPack) {
1510 return DAG->IsReachable(VPack, *TempExp);
1514 unsigned PackPredCount =
1515 std::count_if((*TempMFMA)->Preds.begin(), (*TempMFMA)->Preds.end(),
1516 [&isBitPack](
SDep &Pred) {
1517 auto Opc = Pred.getSUnit()->getInstr()->getOpcode();
1518 return isBitPack(Opc);
1522 std::find_if((*TempMFMA)->Preds.begin(), (*TempMFMA)->Preds.end(),
1523 [&isBitPack](
SDep &Pred) {
1524 auto Opc = Pred.getSUnit()->getInstr()->getOpcode();
1525 return isBitPack(Opc);
1528 if (PackPred == (*TempMFMA)->Preds.end())
1535 std::count_if(PackPred->getSUnit()->Succs.begin(),
1536 PackPred->getSUnit()->Succs.end(), [&
TII](
SDep &Succ) {
1537 return TII->isMFMAorWMMA(*Succ.getSUnit()->getInstr());
1541 MFMAEnablement *= PackSuccCount;
1545 std::count_if(ExpPipeCands.
begin(), ExpPipeCands.
end(),
1546 [
this, &PackPred](
SUnit *ExpBase) {
1547 return DAG->IsReachable(PackPred->getSUnit(), ExpBase);
1550 ExpRequirement *= PackPredCount;
1559 if (
Phase != AMDGPU::SchedulingPhase::PostRA)
1560 MFMAChainSeeds.clear();
1561 if (
Phase != AMDGPU::SchedulingPhase::PostRA && !analyzeDAG(
TII))
1567bool MFMAExpInterleaveOpt::applyIGLPStrategy(
1572 bool IsSmallKernelType =
1573 MFMAEnablement == 2 && ExpRequirement == 4 && TransPipeCount == 32;
1574 bool IsLargeKernelType =
1575 MFMAEnablement == 4 && ExpRequirement == 4 && TransPipeCount == 64;
1577 if (!(IsSmallKernelType || IsLargeKernelType))
1583 unsigned PipelineSyncID = 0;
1584 SchedGroup *SG =
nullptr;
1586 unsigned MFMAChain = 0;
1587 unsigned PositionInChain = 0;
1588 unsigned CurrMFMAForTransPosition = 0;
1590 auto incrementTransPosition = [&MFMAChain, &PositionInChain,
1591 &CurrMFMAForTransPosition]() {
1592 CurrMFMAForTransPosition += MFMAEnablement;
1593 PositionInChain = (CurrMFMAForTransPosition / MFMAChains);
1594 MFMAChain = CurrMFMAForTransPosition % MFMAChains;
1597 auto getNextTransPositionInChain = [&CurrMFMAForTransPosition]() {
1598 auto TempMFMAForTrans = CurrMFMAForTransPosition + MFMAEnablement;
1599 return (TempMFMAForTrans / MFMAChains);
1602 auto getNextTransMFMAChain = [&CurrMFMAForTransPosition]() {
1603 auto TempMFMAForTrans = CurrMFMAForTransPosition + MFMAEnablement;
1604 return TempMFMAForTrans % MFMAChains;
1607 unsigned CurrMFMAPosition = 0;
1608 unsigned MFMAChainForMFMA = 0;
1609 unsigned PositionInChainForMFMA = 0;
1611 auto incrementMFMAPosition = [&CurrMFMAPosition, &MFMAChainForMFMA,
1612 &PositionInChainForMFMA]() {
1614 MFMAChainForMFMA = CurrMFMAPosition % MFMAChains;
1615 PositionInChainForMFMA = CurrMFMAPosition / MFMAChains;
1618 bool IsPostRA =
Phase == AMDGPU::SchedulingPhase::PostRA;
1619 assert(IsPostRA || MFMAChainSeeds.size() == MFMAChains);
1621 bool UsesFMA = IsSmallKernelType || !IsPostRA;
1622 bool UsesDSRead = IsLargeKernelType && !IsPostRA && FirstPipeDSR;
1623 bool UsesCvt = HasCvt && (IsSmallKernelType || !IsPostRA);
1624 bool UsesVALU = IsSmallKernelType;
1629 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1630 SchedGroupMask::VALU, ExpRequirement, PipelineSyncID, DAG,
TII);
1631 if (!IsPostRA && MFMAChains) {
1632 SG->addRule(std::make_shared<EnablesNthMFMAInChain>(
1633 PositionInChain, MFMAChainSeeds[MFMAChain],
TII, SG->getSGID(),
1637 std::make_shared<EnablesNthMFMA>(1,
TII, SG->getSGID(),
true));
1638 SG->addRule(std::make_shared<IsFMA>(
TII, SG->getSGID()));
1639 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1642 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1643 SchedGroupMask::VALU, ExpRequirement, PipelineSyncID, DAG,
TII);
1644 if (!IsPostRA && MFMAChains) {
1645 SG->addRule(std::make_shared<EnablesNthMFMAInChain>(
1646 getNextTransPositionInChain(),
1647 MFMAChainSeeds[getNextTransMFMAChain()],
TII, SG->getSGID(),
true));
1649 SG->addRule(std::make_shared<EnablesNthMFMA>(MFMAEnablement + 1,
TII,
1650 SG->getSGID(),
true));
1651 SG->addRule(std::make_shared<IsFMA>(
TII, SG->getSGID()));
1652 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1656 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1657 SchedGroupMask::DS_READ, 2, PipelineSyncID, DAG,
TII);
1658 SG->addRule(std::make_shared<OccursAtOrAfterNode>(*FirstPipeDSR,
TII,
1660 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1664 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1665 SchedGroupMask::TRANS, ExpRequirement, PipelineSyncID, DAG,
TII);
1666 if (!IsPostRA && MFMAChains)
1667 SG->addRule(std::make_shared<EnablesNthMFMAInChain>(
1668 PositionInChain, MFMAChainSeeds[MFMAChain],
TII, SG->getSGID(),
true));
1670 SG->addRule(std::make_shared<EnablesNthMFMA>(1,
TII, SG->getSGID(),
true));
1671 SG->addRule(std::make_shared<IsPipeExp>(
TII, SG->getSGID(),
true));
1672 SG->addRule(std::make_shared<LessThanNSuccs>(8,
TII, SG->getSGID(),
1673 HasChainBetweenCvt));
1674 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1676 incrementTransPosition();
1679 for (
unsigned I = 0;
I < ExpRequirement;
I++) {
1682 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1683 SchedGroupMask::VALU, 1, PipelineSyncID, DAG,
TII);
1684 SG->addRule(std::make_shared<IsCvt>(
TII, SG->getSGID()));
1685 if (HasChainBetweenCvt)
1686 SG->addRule(std::make_shared<IsReachableFromPrevNthGroup>(
1687 1 + (2 + UsesFMA) *
I,
TII, SG->getSGID()));
1689 SG->addRule(std::make_shared<IsSuccOfPrevNthGroup>(
1690 1 + (2 + UsesFMA) *
I,
TII, SG->getSGID()));
1691 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1696 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1697 SchedGroupMask::VALU, 1, PipelineSyncID, DAG,
TII);
1698 if (!IsPostRA && MFMAChains) {
1699 SG->addRule(std::make_shared<EnablesNthMFMAInChain>(
1700 getNextTransPositionInChain(),
1701 MFMAChainSeeds[getNextTransMFMAChain()],
TII, SG->getSGID(),
true));
1703 SG->addRule(std::make_shared<EnablesNthMFMA>(2 * MFMAEnablement + 1,
1704 TII, SG->getSGID(),
true));
1705 SG->addRule(std::make_shared<IsFMA>(
TII, SG->getSGID()));
1706 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1710 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1711 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG,
TII);
1712 if (!IsPostRA && MFMAChains)
1713 SG->addRule(std::make_shared<EnablesNthMFMAInChain>(
1714 PositionInChain, MFMAChainSeeds[MFMAChain],
TII, SG->getSGID(),
1717 SG->addRule(std::make_shared<EnablesNthMFMA>(MFMAEnablement + 1,
TII,
1718 SG->getSGID(),
true));
1719 SG->addRule(std::make_shared<IsPipeExp>(
TII, SG->getSGID(),
true));
1720 SG->addRule(std::make_shared<LessThanNSuccs>(8,
TII, SG->getSGID(),
1721 HasChainBetweenCvt));
1722 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1727 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1728 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG,
TII);
1729 SG->addRule(std::make_shared<IsPipeExp>(
TII, SG->getSGID(),
true));
1730 SG->addRule(std::make_shared<GreaterThanOrEqualToNSuccs>(
1731 8,
TII, SG->getSGID(), HasChainBetweenCvt));
1732 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1737 unsigned MFMARatio =
1738 MFMAEnablement > ExpRequirement ? MFMAEnablement / ExpRequirement : 1;
1741 MFMAEnablement > ExpRequirement ? 1 : ExpRequirement / MFMAEnablement;
1743 unsigned RemainingExp = TransPipeCount > (2 * ExpRequirement)
1744 ? TransPipeCount - (2 * ExpRequirement)
1746 unsigned ExpLoopCount = RemainingExp / ExpRatio;
1748 unsigned MFMAInLoop = MFMAPipeCount > (MFMAEnablement * 2)
1749 ? MFMAPipeCount - (MFMAEnablement * 2)
1751 unsigned MFMALoopCount = MFMAInLoop / MFMARatio;
1753 AddPipeCount < MFMAPipeCount ? 1 : AddPipeCount / MFMAPipeCount;
1754 unsigned LoopSize = std::min(ExpLoopCount, MFMALoopCount);
1756 for (
unsigned I = 0;
I < LoopSize;
I++) {
1757 if (!(
I * ExpRatio % ExpRequirement))
1758 incrementTransPosition();
1761 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1762 SchedGroupMask::MFMA, MFMARatio, PipelineSyncID, DAG,
TII);
1763 if (!IsPostRA && MFMAChains)
1764 SG->addRule(std::make_shared<IsExactMFMA>(
1765 PositionInChainForMFMA, MFMAChainSeeds[MFMAChainForMFMA],
TII,
1766 SG->getSGID(),
true));
1768 SG->addRule(std::make_shared<OccursAfterExp>(
TII, SG->getSGID(),
true));
1769 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1770 incrementMFMAPosition();
1773 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1774 SchedGroupMask::VALU, VALUOps, PipelineSyncID, DAG,
TII);
1775 SG->addRule(std::make_shared<IsPipeAdd>(
TII, SG->getSGID()));
1776 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1779 if (UsesDSRead && !(
I % 4)) {
1780 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1781 SchedGroupMask::DS_READ, 2, PipelineSyncID, DAG,
TII);
1782 SG->addRule(std::make_shared<OccursAtOrAfterNode>(*FirstPipeDSR,
TII,
1784 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1788 for (
unsigned J = 0; J < ExpRatio; J++) {
1789 auto MFMAOffset = (1 + UsesVALU) * MFMARatio * (
I + 1);
1790 auto MaxMFMAOffset =
1791 (1 + UsesVALU) * ExpRequirement * MFMARatio / ExpRatio;
1795 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1796 SchedGroupMask::VALU, 1, PipelineSyncID, DAG,
TII);
1797 SG->addRule(std::make_shared<IsCvt>(
TII, SG->getSGID()));
1798 auto BaseDiff = (2 + UsesFMA) * (ExpRequirement - 1) + 1;
1799 auto DSROffset =
I / 4 + 1;
1800 auto MaxDSROffset = MaxMFMAOffset / 4;
1802 auto ExpOffset =
I * ExpRatio + J >= ExpRequirement ? 0 : 1;
1803 auto CurrentOffset = UsesDSRead * std::min(MaxDSROffset, DSROffset) +
1804 std::min(MaxMFMAOffset, MFMAOffset) + BaseDiff +
1806 if (HasChainBetweenCvt)
1807 SG->addRule(std::make_shared<IsReachableFromPrevNthGroup>(
1808 CurrentOffset,
TII, SG->getSGID()));
1810 SG->addRule(std::make_shared<IsSuccOfPrevNthGroup>(CurrentOffset,
TII,
1812 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1817 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1818 SchedGroupMask::VALU, 1, PipelineSyncID, DAG,
TII);
1819 if (!IsPostRA && MFMAChains)
1820 SG->addRule(std::make_shared<EnablesNthMFMAInChain>(
1821 getNextTransPositionInChain(),
1822 MFMAChainSeeds[getNextTransMFMAChain()],
TII, SG->getSGID(),
1825 SG->addRule(std::make_shared<EnablesNthMFMA>(
1826 (((
I * ExpRatio + J) / ExpRequirement) + 3) * MFMAEnablement + 1,
1827 TII, SG->getSGID(),
true));
1828 SG->addRule(std::make_shared<IsFMA>(
TII, SG->getSGID()));
1829 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1833 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1834 SchedGroupMask::TRANS, 1, PipelineSyncID, DAG,
TII);
1835 if (!IsPostRA && MFMAChains)
1836 SG->addRule(std::make_shared<EnablesNthMFMAInChain>(
1837 PositionInChain, MFMAChainSeeds[MFMAChain],
TII, SG->getSGID(),
1840 SG->addRule(std::make_shared<EnablesNthMFMA>(
1841 (((
I * ExpRatio + J) / ExpRequirement) + 2) * MFMAEnablement + 1,
1842 TII, SG->getSGID(),
true));
1843 SG->addRule(std::make_shared<IsPipeExp>(
TII, SG->getSGID(),
true));
1844 SG->addRule(std::make_shared<LessThanNSuccs>(8,
TII, SG->getSGID(),
1845 HasChainBetweenCvt));
1846 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1851 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
1852 SchedGroupMask::MFMA, MFMAEnablement * 2, PipelineSyncID, DAG,
TII);
1853 SG->addRule(std::make_shared<OccursAfterExp>(
TII, SG->getSGID(),
true));
1854 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
1858class MFMASmallGemmSingleWaveOpt final :
public IGLPStrategy {
1861 class EnablesInitialMFMA final :
public InstructionRule {
1865 if (!SyncPipe.
size())
1868 if (!Cache->size()) {
1869 for (
auto &Elt : SyncPipe[0].DAG->
SUnits) {
1870 if (
TII->isMFMAorWMMA(*Elt.getInstr())) {
1874 Cache->push_back(&Elt);
1880 auto DAG = SyncPipe[0].DAG;
1881 for (
auto &Elt : *Cache) {
1889 bool NeedsCache =
false)
1890 : InstructionRule(
TII, SGID, NeedsCache) {}
1894 class IsPermForDSW final :
public InstructionRule {
1899 if (
MI->getOpcode() != AMDGPU::V_PERM_B32_e64)
1902 bool FitsInGroup =
false;
1904 if (!Collection.
size()) {
1905 for (
auto &Succ : SU->
Succs) {
1906 SUnit *SuccUnit = Succ.getSUnit();
1909 Cache->push_back(SuccUnit);
1922 return ThisSucc.getSUnit() == Elt;
1927 IsPermForDSW(
const SIInstrInfo *
TII,
unsigned SGID,
bool NeedsCache =
false)
1928 : InstructionRule(
TII, SGID, NeedsCache) {}
1932 class IsSuccOfPrevGroup final :
public InstructionRule {
1936 SchedGroup *OtherGroup =
nullptr;
1937 for (
auto &PipeSG : SyncPipe) {
1938 if ((
unsigned)PipeSG.getSGID() == SGID - 1) {
1939 OtherGroup = &PipeSG;
1945 if (!OtherGroup->Collection.size())
1949 return (std::any_of(OtherGroup->Collection.begin(),
1950 OtherGroup->Collection.end(), [&SU](
SUnit *Elt) {
1951 return std::any_of(Elt->Succs.begin(),
1954 return Succ.getSUnit() == SU;
1959 bool NeedsCache =
false)
1960 : InstructionRule(
TII, SGID, NeedsCache) {}
1964 class VMEMSize final :
public InstructionRule {
1969 if (
MI->getOpcode() == TargetOpcode::BUNDLE)
1971 if (!Collection.
size())
1976 auto TRI =
TII->getRegisterInfo();
1977 auto &
MRI =
MI->getParent()->getParent()->getRegInfo();
1978 for (
auto &Elt : Collection) {
1979 auto Op = Elt->getInstr()->getOperand(0);
1981 TRI.getRegSizeInBits(*
TRI.getRegClassForOperandReg(
MRI,
Op));
1985 if (NumBits < 128) {
1987 if (NumBits +
TRI.getRegSizeInBits(*
TRI.getRegClassForOperandReg(
1988 MRI,
MI->getOperand(0))) <=
1996 VMEMSize(
const SIInstrInfo *
TII,
unsigned SGID,
bool NeedsCache =
false)
1997 : InstructionRule(
TII, SGID, NeedsCache) {}
2002 class SharesPredWithPrevNthGroup final :
public InstructionRule {
2004 unsigned Distance = 1;
2009 SchedGroup *OtherGroup =
nullptr;
2010 if (!SyncPipe.
size())
2013 if (!Cache->size()) {
2015 for (
auto &PipeSG : SyncPipe) {
2016 if ((
unsigned)PipeSG.getSGID() == SGID - Distance) {
2017 OtherGroup = &PipeSG;
2023 if (!OtherGroup->Collection.size())
2026 for (
auto &OtherEle : OtherGroup->Collection) {
2027 for (
auto &Pred : OtherEle->Preds) {
2028 if (Pred.getSUnit()->getInstr()->getOpcode() ==
2029 AMDGPU::V_PERM_B32_e64)
2030 Cache->push_back(Pred.getSUnit());
2039 auto DAG = SyncPipe[0].DAG;
2046 SharesPredWithPrevNthGroup(
unsigned Distance,
const SIInstrInfo *
TII,
2047 unsigned SGID,
bool NeedsCache =
false)
2048 : InstructionRule(
TII, SGID, NeedsCache), Distance(Distance) {}
2052 bool applyIGLPStrategy(
2063 : IGLPStrategy(DAG,
TII) {
2068static unsigned DSWCount = 0;
2069static unsigned DSWWithPermCount = 0;
2070static unsigned DSWWithSharedVMEMCount = 0;
2072bool MFMASmallGemmSingleWaveOpt::applyIGLPStrategy(
2076 unsigned MFMACount = 0;
2077 unsigned DSRCount = 0;
2079 bool IsInitial =
Phase == AMDGPU::SchedulingPhase::Initial;
2081 assert((!IsInitial || (DSWCount == 0 && DSWWithPermCount == 0 &&
2082 DSWWithSharedVMEMCount == 0)) &&
2083 "DSWCounters should be zero in pre-RA scheduling!");
2085 for (
auto &SU : DAG->SUnits) {
2086 auto I = SU.getInstr();
2087 if (
TII->isMFMAorWMMA(*
I))
2089 else if (
TII->isDS(*
I)) {
2092 else if (
I->mayStore() && IsInitial) {
2094 for (
auto Pred : SU.Preds) {
2095 if (Pred.getSUnit()->getInstr()->getOpcode() ==
2096 AMDGPU::V_PERM_B32_e64) {
2106 DSWWithPermCount = DSWithPerms.
size();
2107 auto I = DSWithPerms.
begin();
2108 auto E = DSWithPerms.
end();
2118 for (;
I != E;
I++) {
2119 SUnit *Cand =
nullptr;
2120 bool MissedAny =
false;
2121 for (
auto &Pred : (*I)->Preds) {
2122 if (Pred.getSUnit()->getInstr()->getOpcode() != AMDGPU::V_PERM_B32_e64)
2128 for (
auto &Succ : Pred.getSUnit()->Succs) {
2129 auto MI = Succ.getSUnit()->getInstr();
2130 if (!
TII->isVMEM(*
MI) || !
MI->mayLoad())
2133 if (MissedAny || !VMEMLookup.
size()) {
2135 VMEMLookup[
MI] = *
I;
2141 VMEMLookup[
MI] = *
I;
2145 Cand = VMEMLookup[
MI];
2152 if (!MissedAny && Cand) {
2153 DSWWithSharedVMEMCount += 2;
2160 assert(DSWWithSharedVMEMCount <= DSWWithPermCount);
2162 unsigned PipelineSyncID = 0;
2164 if (DSWWithPermCount) {
2165 for (
unsigned I = 0;
I < MFMACount;
I++) {
2166 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2167 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG,
TII);
2168 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2170 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2171 SchedGroupMask::VALU, 2, PipelineSyncID, DAG,
TII);
2172 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2182 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2183 SchedGroupMask::DS_READ, 4, PipelineSyncID, DAG,
TII);
2184 SG->addRule(std::make_shared<EnablesInitialMFMA>(
TII, SG->getSGID(),
true));
2185 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2187 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2188 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG,
TII);
2189 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2192 for (
unsigned I = 0;
I < DSRCount - 4; ++
I) {
2193 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2194 SchedGroupMask::DS_READ, 1, PipelineSyncID, DAG,
TII);
2195 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2197 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2198 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG,
TII);
2199 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2205 for (
unsigned I = 0;
I < DSWWithPermCount - DSWWithSharedVMEMCount; ++
I) {
2206 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2207 SchedGroupMask::VALU, 4, PipelineSyncID, DAG,
TII);
2208 SG->addRule(std::make_shared<IsPermForDSW>(
TII, SG->getSGID(),
true));
2209 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2211 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2212 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG,
TII);
2213 SG->addRule(std::make_shared<IsSuccOfPrevGroup>(
TII, SG->getSGID()));
2214 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2216 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2217 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG,
TII);
2218 SG->addRule(std::make_shared<SharesPredWithPrevNthGroup>(
2219 1,
TII, SG->getSGID(),
true));
2220 SG->addRule(std::make_shared<VMEMSize>(
TII, SG->getSGID()));
2221 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2223 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2224 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG,
TII);
2225 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2227 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2228 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG,
TII);
2229 SG->addRule(std::make_shared<SharesPredWithPrevNthGroup>(
2230 3,
TII, SG->getSGID(),
true));
2231 SG->addRule(std::make_shared<VMEMSize>(
TII, SG->getSGID()));
2232 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2234 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2235 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG,
TII);
2236 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2242 for (
unsigned I = 0;
I < DSWCount - DSWWithPermCount;
I++) {
2243 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2244 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG,
TII);
2245 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2247 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2248 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG,
TII);
2249 SG->addRule(std::make_shared<VMEMSize>(
TII, SG->getSGID()));
2250 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2252 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2253 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG,
TII);
2254 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2262 for (
unsigned I = 0;
I < DSWWithSharedVMEMCount; ++
I) {
2263 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2264 SchedGroupMask::VALU, 4, PipelineSyncID, DAG,
TII);
2265 SG->addRule(std::make_shared<IsPermForDSW>(
TII, SG->getSGID(),
true));
2266 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2268 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2269 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG,
TII);
2270 SG->addRule(std::make_shared<IsSuccOfPrevGroup>(
TII, SG->getSGID()));
2271 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2273 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2274 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG,
TII);
2275 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2277 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2278 SchedGroupMask::VALU, 4, PipelineSyncID, DAG,
TII);
2279 SG->addRule(std::make_shared<IsPermForDSW>(
TII, SG->getSGID(),
true));
2280 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2282 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2283 SchedGroupMask::DS_WRITE, 1, PipelineSyncID, DAG,
TII);
2284 SG->addRule(std::make_shared<IsSuccOfPrevGroup>(
TII, SG->getSGID()));
2285 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2287 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2288 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG,
TII);
2289 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2291 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2292 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG,
TII);
2293 SG->addRule(std::make_shared<SharesPredWithPrevNthGroup>(
2294 2,
TII, SG->getSGID(),
true));
2295 SG->addRule(std::make_shared<VMEMSize>(
TII, SG->getSGID()));
2296 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2298 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2299 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG,
TII);
2300 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2302 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2303 SchedGroupMask::VMEM_READ, 4, PipelineSyncID, DAG,
TII);
2304 SG->addRule(std::make_shared<SharesPredWithPrevNthGroup>(
2305 4,
TII, SG->getSGID(),
true));
2306 SG->addRule(std::make_shared<VMEMSize>(
TII, SG->getSGID()));
2307 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2309 SG = &SyncedSchedGroups[PipelineSyncID].emplace_back(
2310 SchedGroupMask::MFMA, 1, PipelineSyncID, DAG,
TII);
2311 SG->initSchedGroup(SyncedInstrs[SG->getSyncID()]);
2317static std::unique_ptr<IGLPStrategy>
2321 case MFMASmallGemmOptID:
2322 return std::make_unique<MFMASmallGemmOpt>(DAG,
TII);
2323 case MFMASmallGemmSingleWaveOptID:
2324 return std::make_unique<MFMASmallGemmSingleWaveOpt>(DAG,
TII);
2325 case MFMAExpInterleave:
2326 return std::make_unique<MFMAExpInterleaveOpt>(DAG,
TII);
2347 void addSchedBarrierEdges(
SUnit &SU);
2358 SchedGroupMask invertSchedBarrierMask(SchedGroupMask Mask)
const;
2361 void initSchedGroupBarrierPipelineStage(
2362 std::vector<SUnit>::reverse_iterator RIter);
2364 bool initIGLPOpt(
SUnit &SU);
2374 bool IsBottomUp = 1;
2379 IGroupLPDAGMutation() =
default;
2383unsigned SchedGroup::NumSchedGroups = 0;
2395 if (
MI.isMetaInstruction())
2398 else if (((SGMask & SchedGroupMask::ALU) != SchedGroupMask::NONE) &&
2403 else if (((SGMask & SchedGroupMask::VALU) != SchedGroupMask::NONE) &&
2407 else if (((SGMask & SchedGroupMask::SALU) != SchedGroupMask::NONE) &&
2411 else if (((SGMask & SchedGroupMask::MFMA) != SchedGroupMask::NONE) &&
2412 TII->isMFMAorWMMA(
MI))
2415 else if (((SGMask & SchedGroupMask::VMEM) != SchedGroupMask::NONE) &&
2419 else if (((SGMask & SchedGroupMask::VMEM_READ) != SchedGroupMask::NONE) &&
2424 else if (((SGMask & SchedGroupMask::VMEM_WRITE) != SchedGroupMask::NONE) &&
2429 else if (((SGMask & SchedGroupMask::DS) != SchedGroupMask::NONE) &&
2433 else if (((SGMask & SchedGroupMask::DS_READ) != SchedGroupMask::NONE) &&
2434 MI.mayLoad() &&
TII->isDS(
MI))
2437 else if (((SGMask & SchedGroupMask::DS_WRITE) != SchedGroupMask::NONE) &&
2438 MI.mayStore() &&
TII->isDS(
MI))
2441 else if (((SGMask & SchedGroupMask::TRANS) != SchedGroupMask::NONE) &&
2446 dbgs() <<
"For SchedGroup with mask " <<
format_hex((
int)SGMask, 10,
true)
2447 << (Result ?
" could classify " :
" unable to classify ") <<
MI);
2452int SchedGroup::link(
SUnit &SU,
bool MakePred,
2453 std::vector<std::pair<SUnit *, SUnit *>> &AddedEdges) {
2454 int MissedEdges = 0;
2455 for (
auto *
A : Collection) {
2457 if (
A ==
B ||
A->getInstr()->getOpcode() == AMDGPU::SCHED_GROUP_BARRIER)
2467 bool Added = tryAddEdge(
A,
B);
2469 AddedEdges.push_back(std::pair(
A,
B));
2477void SchedGroup::link(
SUnit &SU,
bool MakePred) {
2478 for (
auto *
A : Collection) {
2480 if (
A->getInstr()->getOpcode() == AMDGPU::SCHED_GROUP_BARRIER)
2489void SchedGroup::link(
SUnit &SU,
2491 for (
auto *
A : Collection) {
2500void SchedGroup::link(SchedGroup &OtherGroup) {
2501 for (
auto *
B : OtherGroup.Collection)
2505bool SchedGroup::canAddSU(
SUnit &SU)
const {
2507 if (
MI.getOpcode() != TargetOpcode::BUNDLE)
2508 return canAddMI(
MI);
2513 while (E !=
MBB->
end() && E->isBundledWithPred())
2520void SchedGroup::initSchedGroup() {
2521 for (
auto &SU : DAG->
SUnits) {
2530void SchedGroup::initSchedGroup(std::vector<SUnit>::reverse_iterator RIter,
2531 SUnitsToCandidateSGsMap &SyncedInstrs) {
2532 SUnit &InitSU = *RIter;
2533 for (
auto E = DAG->
SUnits.rend(); RIter != E; ++RIter) {
2539 SyncedInstrs[&SU].push_back(SGID);
2547void SchedGroup::initSchedGroup(SUnitsToCandidateSGsMap &SyncedInstrs) {
2548 auto I = DAG->
SUnits.rbegin();
2549 auto E = DAG->
SUnits.rend();
2550 for (;
I != E; ++
I) {
2555 SyncedInstrs[&SU].push_back(SGID);
2561 if (!TSchedModel || DAGInstrs->
SUnits.empty())
2566 TII =
ST.getInstrInfo();
2568 SyncedSchedGroups.clear();
2569 SyncedInstrs.clear();
2570 bool FoundSB =
false;
2571 bool FoundIGLP =
false;
2572 bool ShouldApplyIGLP =
false;
2573 for (
auto R = DAG->
SUnits.rbegin(), E = DAG->
SUnits.rend();
R != E; ++
R) {
2574 unsigned Opc =
R->getInstr()->getOpcode();
2576 if (Opc == AMDGPU::SCHED_BARRIER) {
2577 addSchedBarrierEdges(*R);
2579 }
else if (Opc == AMDGPU::SCHED_GROUP_BARRIER) {
2580 initSchedGroupBarrierPipelineStage(R);
2582 }
else if (Opc == AMDGPU::IGLP_OPT) {
2583 resetEdges(*R, DAG);
2584 if (!FoundSB && !FoundIGLP) {
2586 ShouldApplyIGLP = initIGLPOpt(*R);
2591 if (FoundSB || (FoundIGLP && ShouldApplyIGLP)) {
2592 PipelineSolver PS(SyncedSchedGroups, SyncedInstrs, DAG, IsBottomUp);
2600void IGroupLPDAGMutation::addSchedBarrierEdges(
SUnit &SchedBarrier) {
2602 assert(
MI.getOpcode() == AMDGPU::SCHED_BARRIER);
2605 resetEdges(SchedBarrier, DAG);
2606 LLVM_DEBUG(
dbgs() <<
"Building SchedGroup for SchedBarrier with Mask: "
2607 <<
MI.getOperand(0).getImm() <<
"\n");
2609 invertSchedBarrierMask((SchedGroupMask)
MI.getOperand(0).getImm());
2610 SchedGroup SG(InvertedMask, std::nullopt, DAG,
TII);
2611 SG.initSchedGroup();
2617 const SUnit *
A,
const SUnit *
B) {
return A->NodeNum >
B->NodeNum; });
2621IGroupLPDAGMutation::invertSchedBarrierMask(SchedGroupMask Mask)
const {
2624 SchedGroupMask InvertedMask = ~Mask;
2627 if ((InvertedMask & SchedGroupMask::ALU) == SchedGroupMask::NONE)
2628 InvertedMask &= ~SchedGroupMask::VALU & ~SchedGroupMask::SALU &
2629 ~SchedGroupMask::MFMA & ~SchedGroupMask::TRANS;
2631 else if ((InvertedMask & SchedGroupMask::VALU) == SchedGroupMask::NONE ||
2632 (InvertedMask & SchedGroupMask::SALU) == SchedGroupMask::NONE ||
2633 (InvertedMask & SchedGroupMask::MFMA) == SchedGroupMask::NONE ||
2634 (InvertedMask & SchedGroupMask::TRANS) == SchedGroupMask::NONE)
2635 InvertedMask &= ~SchedGroupMask::ALU;
2638 if ((InvertedMask & SchedGroupMask::VMEM) == SchedGroupMask::NONE)
2639 InvertedMask &= ~SchedGroupMask::VMEM_READ & ~SchedGroupMask::VMEM_WRITE;
2641 else if ((InvertedMask & SchedGroupMask::VMEM_READ) == SchedGroupMask::NONE ||
2642 (InvertedMask & SchedGroupMask::VMEM_WRITE) == SchedGroupMask::NONE)
2643 InvertedMask &= ~SchedGroupMask::VMEM;
2646 if ((InvertedMask & SchedGroupMask::DS) == SchedGroupMask::NONE)
2647 InvertedMask &= ~SchedGroupMask::DS_READ & ~SchedGroupMask::DS_WRITE;
2649 else if ((InvertedMask & SchedGroupMask::DS_READ) == SchedGroupMask::NONE ||
2650 (InvertedMask & SchedGroupMask::DS_WRITE) == SchedGroupMask::NONE)
2651 InvertedMask &= ~SchedGroupMask::DS;
2653 LLVM_DEBUG(
dbgs() <<
"After Inverting, SchedGroup Mask: " << (
int)InvertedMask
2656 return InvertedMask;
2659void IGroupLPDAGMutation::initSchedGroupBarrierPipelineStage(
2660 std::vector<SUnit>::reverse_iterator RIter) {
2663 resetEdges(*RIter, DAG);
2670 auto &SG = SyncedSchedGroups[SyncID].emplace_back((SchedGroupMask)SGMask,
2673 SG.initSchedGroup(RIter, SyncedInstrs[SG.getSyncID()]);
2676bool IGroupLPDAGMutation::initIGLPOpt(
SUnit &SU) {
2677 IGLPStrategyID StrategyID =
2679 auto S = createIGLPStrategy(StrategyID, DAG,
TII);
2680 if (!S->shouldApplyStrategy(DAG,
Phase))
2683 IsBottomUp = S->IsBottomUp;
2684 return S->applyIGLPStrategy(SyncedInstrs, SyncedSchedGroups,
Phase);
2696std::unique_ptr<ScheduleDAGMutation>
2698 return std::make_unique<IGroupLPDAGMutation>(
Phase);
unsigned const MachineRegisterInfo * MRI
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
Provides AMDGPU specific target descriptions.
The AMDGPU TargetMachine interface definition for hw codegen targets.
#define LLVM_MARK_AS_BITMASK_ENUM(LargestValue)
LLVM_MARK_AS_BITMASK_ENUM lets you opt in an individual enum type so you can perform bitwise operatio...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file defines the DenseMap class.
const HexagonInstrInfo * TII
unsigned const TargetRegisterInfo * TRI
Interface definition for SIInstrInfo.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
This class represents an Operation in the Expression.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Instructions::iterator instr_iterator
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const MachineOperand & getOperand(unsigned i) const
@ Data
Regular data dependence (aka true-dependence).
@ Artificial
Arbitrary strong DAG edge (no real dependence).
Scheduling unit. This is a node in the scheduling DAG.
unsigned NodeNum
Entry # of node in the node vector.
void removePred(const SDep &D)
Removes the specified edge as a pred of the current node if it exists.
SmallVector< SDep, 4 > Succs
All sunit successors.
SmallVector< SDep, 4 > Preds
All sunit predecessors.
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
A ScheduleDAG for scheduling lists of MachineInstr.
const TargetSchedModel * getSchedModel() const
Gets the machine model for instruction scheduling.
bool addEdge(SUnit *SuccSU, const SDep &PredDep)
Add a DAG edge to the given SU with the given predecessor dependence data.
bool IsReachable(SUnit *SU, SUnit *TargetSU)
IsReachable - Checks if SU is reachable from TargetSU.
bool canAddEdge(SUnit *SuccSU, SUnit *PredSU)
True if an edge can be added from PredSU to SuccSU without creating a cycle.
void dump() const override
ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...
Mutate the DAG as a postpass after normal DAG building.
virtual void apply(ScheduleDAGInstrs *DAG)=0
std::vector< SUnit > SUnits
The scheduling units.
MachineFunction & MF
Machine function.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
reverse_iterator rbegin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Provide an instruction scheduling machine model to CodeGen passes.
An efficient, type-erasing, non-owning reference to a callable.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
initializer< Ty > init(const Ty &Val)
void link(std::unique_ptr< LinkGraph > G, std::unique_ptr< JITLinkContext > Ctx)
Link the given graph.
This is an optimization pass for GlobalISel generic memory operations.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
std::unique_ptr< ScheduleDAGMutation > createIGroupLPDAGMutation(AMDGPU::SchedulingPhase Phase)
Phase specifes whether or not this is a reentry into the IGroupLPDAGMutation.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FormattedNumber format_hex(uint64_t N, unsigned Width, bool Upper=false)
format_hex - Output N as a fixed width hexadecimal.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.