30 #define DEBUG_TYPE "post-RA-sched"
35 cl::desc(
"Debug control for aggressive anti-dep breaker"),
39 cl::desc(
"Debug control for aggressive anti-dep breaker"),
44 NumTargetRegs(TargetRegs), GroupNodes(TargetRegs, 0),
45 GroupNodeIndices(TargetRegs, 0),
46 KillIndices(TargetRegs, 0),
47 DefIndices(TargetRegs, 0)
49 const unsigned BBSize = BB->
size();
50 for (
unsigned i = 0;
i < NumTargetRegs; ++
i) {
53 GroupNodeIndices[
i] =
i;
56 DefIndices[
i] = BBSize;
61 unsigned Node = GroupNodeIndices[
Reg];
62 while (GroupNodes[Node] != Node)
63 Node = GroupNodes[Node];
70 std::vector<unsigned> &
Regs,
71 std::multimap<unsigned, AggressiveAntiDepState::RegisterReference> *RegRefs)
73 for (
unsigned Reg = 0;
Reg != NumTargetRegs; ++
Reg) {
81 assert(GroupNodes[0] == 0 &&
"GroupNode 0 not parent!");
82 assert(GroupNodeIndices[0] == 0 &&
"Reg 0 not in Group 0!");
89 unsigned Parent = (Group1 == 0) ? Group1 : Group2;
90 unsigned Other = (Parent == Group1) ? Group2 : Group1;
91 GroupNodes.at(Other) = Parent;
100 unsigned idx = GroupNodes.size();
101 GroupNodes.push_back(idx);
102 GroupNodeIndices[
Reg] = idx;
110 return((KillIndices[Reg] != ~0u) && (DefIndices[Reg] == ~0u));
117 TII(MF.getSubtarget().getInstrInfo()),
118 TRI(MF.getSubtarget().getRegisterInfo()), RegClassInfo(RCI),
122 for (
unsigned i = 0, e = CriticalPathRCs.
size();
i < e; ++
i) {
124 if (CriticalPathSet.
none())
125 CriticalPathSet = CPSet;
127 CriticalPathSet |= CPSet;
130 DEBUG(
dbgs() <<
"AntiDep Critical-Path Registers:");
152 for (
const auto &LI : (*SI)->liveins()) {
157 DefIndices[
Reg] = ~0u;
168 if (!IsReturnBlock && !Pristine.
test(Reg))
continue;
170 unsigned AliasReg = *AI;
172 KillIndices[AliasReg] = BB->
size();
173 DefIndices[AliasReg] = ~0u;
184 unsigned InsertPosIndex) {
185 assert(Count < InsertPosIndex &&
"Instruction index out of expected range!");
187 std::set<unsigned> PassthruRegs;
188 GetPassthruRegs(MI, PassthruRegs);
189 PrescanInstruction(MI, Count, PassthruRegs);
190 ScanInstruction(MI, Count);
209 }
else if ((DefIndices[
Reg] < InsertPosIndex)
210 && (DefIndices[
Reg] >= Count)) {
211 DefIndices[
Reg] = Count;
235 void AggressiveAntiDepBreaker::GetPassthruRegs(
239 if (!MO.
isReg())
continue;
241 IsImplicitDefUse(MI, MO)) {
242 const unsigned Reg = MO.
getReg();
245 PassthruRegs.insert(*SubRegs);
257 if (RegSet.
insert(
P->getReg()).second)
258 Edges.push_back(&*
P);
266 const SDep *Next =
nullptr;
267 unsigned NextDepth = 0;
272 const SUnit *PredSU =
P->getSUnit();
273 unsigned PredLatency =
P->getLatency();
274 unsigned PredTotalLatency = PredSU->
getDepth() + PredLatency;
277 if (NextDepth < PredTotalLatency ||
278 (NextDepth == PredTotalLatency &&
P->getKind() ==
SDep::Anti)) {
279 NextDepth = PredTotalLatency;
285 return (Next) ? Next->
getSUnit() :
nullptr;
288 void AggressiveAntiDepBreaker::HandleLastUse(
unsigned Reg,
unsigned KillIdx,
291 const char *footer) {
294 std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>&
303 DEBUG(
if (!header && footer)
dbgs() << footer);
307 if (!State->
IsLive(Reg)) {
308 KillIndices[
Reg] = KillIdx;
309 DefIndices[
Reg] = ~0u;
313 dbgs() << header << TRI->
getName(Reg); header =
nullptr; });
320 unsigned SubregReg = *SubRegs;
321 if (!State->
IsLive(SubregReg)) {
322 KillIndices[SubregReg] = KillIdx;
323 DefIndices[SubregReg] = ~0u;
324 RegRefs.erase(SubregReg);
327 dbgs() << header << TRI->
getName(Reg); header =
nullptr; });
334 DEBUG(
if (!header && footer)
dbgs() << footer);
337 void AggressiveAntiDepBreaker::PrescanInstruction(
338 MachineInstr &MI,
unsigned Count, std::set<unsigned> &PassthruRegs) {
340 std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>&
351 unsigned Reg = MO.
getReg();
352 if (Reg == 0)
continue;
354 HandleLastUse(Reg, Count + 1,
"",
"\tDead Def: ",
"\n");
361 unsigned Reg = MO.
getReg();
362 if (Reg == 0)
continue;
380 unsigned AliasReg = *AI;
381 if (State->
IsLive(AliasReg)) {
384 TRI->
getName(AliasReg) <<
")");
393 RegRefs.insert(std::make_pair(Reg, RR));
403 unsigned Reg = MO.
getReg();
404 if (Reg == 0)
continue;
406 if (MI.
isKill() || (PassthruRegs.count(Reg) != 0))
417 if (TRI->isSuperRegister(Reg, *AI) && State->
IsLive(*AI))
420 DefIndices[*AI] = Count;
425 void AggressiveAntiDepBreaker::ScanInstruction(
MachineInstr &MI,
428 std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>&
456 unsigned Reg = MO.
getReg();
457 if (Reg == 0)
continue;
459 DEBUG(
dbgs() <<
" " << TRI->getName(Reg) <<
"=g" <<
465 HandleLastUse(Reg, Count,
"(last-use)");
477 RegRefs.insert(std::make_pair(Reg, RR));
487 unsigned FirstReg = 0;
490 if (!MO.
isReg())
continue;
491 unsigned Reg = MO.
getReg();
492 if (Reg == 0)
continue;
495 DEBUG(
dbgs() <<
"=" << TRI->getName(Reg));
498 DEBUG(
dbgs() <<
" " << TRI->getName(Reg));
507 BitVector AggressiveAntiDepBreaker::GetRenameRegisters(
unsigned Reg) {
518 BitVector RCBV = TRI->getAllocatableSet(MF, RC);
526 DEBUG(
dbgs() <<
" " << TRI->getRegClassName(RC));
532 bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
533 unsigned AntiDepGroupIndex,
534 RenameOrderType& RenameOrder,
535 std::map<unsigned, unsigned> &RenameMap) {
538 std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>&
544 std::vector<unsigned>
Regs;
546 assert(Regs.size() > 0 &&
"Empty register group!");
547 if (Regs.size() == 0)
553 DEBUG(
dbgs() <<
"\tRename Candidates for Group g" << AntiDepGroupIndex
555 std::map<unsigned, BitVector> RenameRegisterMap;
556 unsigned SuperReg = 0;
557 for (
unsigned i = 0, e = Regs.size();
i != e; ++
i) {
558 unsigned Reg = Regs[
i];
559 if ((SuperReg == 0) || TRI->isSuperRegister(SuperReg, Reg))
563 if (RegRefs.count(Reg) > 0) {
564 DEBUG(
dbgs() <<
"\t\t" << TRI->getName(Reg) <<
":");
568 BV = GetRenameRegisters(Reg);
572 for (
int r = BV.find_first(); r != -1; r = BV.find_next(r))
573 dbgs() <<
" " << TRI->getName(r);
580 for (
unsigned i = 0, e = Regs.size();
i != e; ++
i) {
581 unsigned Reg = Regs[
i];
582 if (Reg == SuperReg)
continue;
583 bool IsSub = TRI->isSubRegister(SuperReg, Reg);
594 static int renamecnt = 0;
598 dbgs() <<
"*** Performing rename " << TRI->getName(SuperReg) <<
612 TRI->getMinimalPhysRegClass(SuperReg,
MVT::Other);
616 DEBUG(
dbgs() <<
"\tEmpty Super Regclass!!\n");
622 RenameOrder.insert(RenameOrderType::value_type(SuperRC, Order.
size()));
624 unsigned OrigR = RenameOrder[SuperRC];
625 unsigned EndR = ((OrigR == Order.
size()) ? 0 : OrigR);
628 if (R == 0) R = Order.
size();
630 const unsigned NewSuperReg = Order[R];
632 if (!
MRI.isAllocatable(NewSuperReg))
continue;
634 if (NewSuperReg == SuperReg)
continue;
636 DEBUG(
dbgs() <<
" [" << TRI->getName(NewSuperReg) <<
':');
642 for (
unsigned i = 0, e = Regs.size();
i != e; ++
i) {
643 unsigned Reg = Regs[
i];
645 if (Reg == SuperReg) {
646 NewReg = NewSuperReg;
648 unsigned NewSubRegIdx = TRI->getSubRegIndex(SuperReg, Reg);
649 if (NewSubRegIdx != 0)
650 NewReg = TRI->getSubReg(NewSuperReg, NewSubRegIdx);
653 DEBUG(
dbgs() <<
" " << TRI->getName(NewReg));
656 if (!RenameRegisterMap[Reg].test(NewReg)) {
665 if (State->
IsLive(NewReg) || (KillIndices[
Reg] > DefIndices[NewReg])) {
671 unsigned AliasReg = *AI;
672 if (State->
IsLive(AliasReg) ||
673 (KillIndices[
Reg] > DefIndices[AliasReg])) {
674 DEBUG(
dbgs() <<
"(alias " << TRI->getName(AliasReg) <<
" live)");
685 for (
const auto &Q :
make_range(RegRefs.equal_range(Reg))) {
700 for (
const auto &Q :
make_range(RegRefs.equal_range(Reg))) {
701 if (!Q.second.Operand->isDef() || !Q.second.Operand->isEarlyClobber())
712 RenameMap.insert(std::pair<unsigned, unsigned>(Reg, NewReg));
717 RenameOrder.erase(SuperRC);
718 RenameOrder.insert(RenameOrderType::value_type(SuperRC, R));
736 const std::vector<SUnit>& SUnits,
739 unsigned InsertPosIndex,
744 std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>&
749 if (SUnits.empty())
return 0;
752 RenameOrderType RenameOrder;
755 std::map<MachineInstr *, const SUnit *> MISUnitMap;
756 for (
unsigned i = 0, e = SUnits.size();
i != e; ++
i) {
757 const SUnit *SU = &SUnits[
i];
758 MISUnitMap.insert(std::pair<MachineInstr *, const SUnit *>(SU->
getInstr(),
765 const SUnit *CriticalPathSU =
nullptr;
767 if (CriticalPathSet.
any()) {
768 for (
unsigned i = 0, e = SUnits.size();
i != e; ++
i) {
769 const SUnit *SU = &SUnits[
i];
770 if (!CriticalPathSU ||
777 CriticalPathMI = CriticalPathSU->
getInstr();
781 DEBUG(
dbgs() <<
"\n===== Aggressive anti-dependency breaking\n");
783 for (
unsigned Reg = 0; Reg < TRI->getNumRegs(); ++
Reg) {
785 DEBUG(
dbgs() <<
" " << TRI->getName(Reg));
796 unsigned Count = InsertPosIndex - 1;
807 std::set<unsigned> PassthruRegs;
808 GetPassthruRegs(MI, PassthruRegs);
811 PrescanInstruction(MI, Count, PassthruRegs);
815 std::vector<const SDep *> Edges;
816 const SUnit *PathSU = MISUnitMap[&
MI];
822 if (&MI == CriticalPathMI) {
824 CriticalPathMI = (CriticalPathSU) ? CriticalPathSU->
getInstr() :
nullptr;
825 }
else if (CriticalPathSet.
any()) {
826 ExcludeRegs = &CriticalPathSet;
833 for (
unsigned i = 0, e = Edges.size();
i != e; ++
i) {
834 const SDep *Edge = Edges[
i];
840 unsigned AntiDepReg = Edge->
getReg();
841 DEBUG(
dbgs() <<
"\tAntidep reg: " << TRI->getName(AntiDepReg));
842 assert(AntiDepReg != 0 &&
"Anti-dependence on reg0?");
844 if (!
MRI.isAllocatable(AntiDepReg)) {
848 }
else if (ExcludeRegs && ExcludeRegs->
test(AntiDepReg)) {
851 DEBUG(
dbgs() <<
" (not critical-path)\n");
853 }
else if (PassthruRegs.count(AntiDepReg) != 0) {
862 assert(AntiDepOp &&
"Can't find index for defined register operand");
878 PE = PathSU->
Preds.end();
P != PE; ++
P) {
879 if (
P->getSUnit() == NextSU ?
880 (
P->getKind() !=
SDep::Anti ||
P->getReg() != AntiDepReg) :
881 (
P->getKind() ==
SDep::Data &&
P->getReg() == AntiDepReg)) {
887 PE = PathSU->
Preds.end();
P != PE; ++
P) {
888 if ((
P->getSUnit() == NextSU) && (
P->getKind() !=
SDep::Anti) &&
893 }
else if ((
P->getSUnit() != NextSU) &&
895 (
P->getReg() == AntiDepReg)) {
896 DEBUG(
dbgs() <<
" (other dependency)\n");
902 if (AntiDepReg == 0)
continue;
919 if (R == AntiDepReg || TRI->isSubRegister(AntiDepReg, R))
925 if (AntiDepReg == 0)
continue;
929 if (AntiDepReg == 0)
continue;
932 const unsigned GroupIndex = State->
GetGroup(AntiDepReg);
933 if (GroupIndex == 0) {
941 std::map<unsigned, unsigned> RenameMap;
942 if (FindSuitableFreeRegisters(GroupIndex, RenameOrder, RenameMap)) {
943 DEBUG(
dbgs() <<
"\tBreaking anti-dependence edge on "
944 << TRI->getName(AntiDepReg) <<
":");
947 for (std::map<unsigned, unsigned>::iterator
948 S = RenameMap.begin(),
E = RenameMap.end(); S !=
E; ++S) {
949 unsigned CurrReg = S->first;
950 unsigned NewReg = S->second;
952 DEBUG(
dbgs() <<
" " << TRI->getName(CurrReg) <<
"->" <<
953 TRI->getName(NewReg) <<
"(" <<
954 RegRefs.count(CurrReg) <<
" refs)");
958 for (
const auto &Q :
make_range(RegRefs.equal_range(CurrReg))) {
959 Q.second.Operand->setReg(NewReg);
963 const SUnit *SU = MISUnitMap[Q.second.Operand->getParent()];
965 for (DbgValueVector::iterator DVI = DbgValues.begin(),
966 DVE = DbgValues.end(); DVI != DVE; ++DVI)
967 if (DVI->second == Q.second.Operand->getParent())
975 RegRefs.erase(NewReg);
976 DefIndices[NewReg] = DefIndices[CurrReg];
977 KillIndices[NewReg] = KillIndices[CurrReg];
980 RegRefs.erase(CurrReg);
981 DefIndices[CurrReg] = KillIndices[CurrReg];
982 KillIndices[CurrReg] = ~0u;
983 assert(((KillIndices[CurrReg] == ~0u) !=
984 (DefIndices[CurrReg] == ~0u)) &&
985 "Kill and Def maps aren't consistent for AntiDepReg!");
994 ScanInstruction(MI, Count);
int find_first() const
find_first - Returns the index of the first set bit, -1 if none of the bits are set.
Information about a register reference within a liverange.
BitVector getPristineRegs(const MachineFunction &MF) const
Return a set of physical registers that are pristine.
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
MachineOperand * findRegisterDefOperand(unsigned Reg, bool isDead=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterDefOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
bool none() const
none - Returns true if none of the bits are set.
int find_next(unsigned Prev) const
find_next - Returns the index of the next set bit following the "Prev" bit.
AggressiveAntiDepState(const unsigned TargetRegs, MachineBasicBlock *BB)
MachineInstr * getInstr() const
getInstr - Return the representative MachineInstr for this SUnit.
MachineOperand * findRegisterUseOperand(unsigned Reg, bool isKill=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool any() const
any - Returns true if any bit is set.
Kind
Kind - These are the different kinds of scheduling dependencies.
SmallVector< SDep, 4 > Preds
unsigned LeaveGroup(unsigned Reg)
A register anti-dependedence (aka WAR).
void StartBlock(MachineBasicBlock *BB) override
Initialize anti-dep breaking for a new basic block.
AggressiveAntiDepBreaker(MachineFunction &MFi, const RegisterClassInfo &RCI, TargetSubtargetInfo::RegClassVector &CriticalPathRCs)
unsigned GetGroup(unsigned Reg)
const HexagonInstrInfo * TII
Kind getKind() const
getKind - Return an enum value representing the kind of the dependence.
This class works in conjunction with the post-RA scheduler to rename registers to break register anti...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Regular data dependence (aka true-dependence).
~AggressiveAntiDepBreaker() override
std::vector< MachineBasicBlock * >::iterator succ_iterator
Reg
All possible values of the reg field in the ModR/M byte.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
ArrayRef< MCPhysReg > getOrder(const TargetRegisterClass *RC) const
getOrder - Returns the preferred allocation order for RC.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
unsigned getNumRegs() const
Return the number of registers this target has (useful for sizing arrays holding per register informa...
A register output-dependence (aka WAW).
static void AntiDepEdges(const SUnit *SU, std::vector< const SDep * > &Edges)
AntiDepEdges - Return in Edges the anti- and output- dependencies in SU that we want to consider for ...
bool hasExtraSrcRegAllocReq(QueryType Type=AnyInBundle) const
Returns true if this instruction source operands have special register allocation requirements that a...
void GetGroupRegs(unsigned Group, std::vector< unsigned > &Regs, std::multimap< unsigned, AggressiveAntiDepState::RegisterReference > *RegRefs)
unsigned UnionGroups(unsigned Reg1, unsigned Reg2)
BitVector getAllocatableSet(const MachineFunction &MF, const TargetRegisterClass *RC=nullptr) const
Returns a bitset indexed by register number indicating if a register is allocatable or not...
virtual const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const =0
Return a null-terminated list of all of the callee-saved registers on this target.
size_t size() const
size - Get the array size.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
const MachineBasicBlock * getParent() const
bool isDebugValue() const
SDep - Scheduling dependency.
bool isEarlyClobber() const
initializer< Ty > init(const Ty &Val)
unsigned BreakAntiDependencies(const std::vector< SUnit > &SUnits, MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, unsigned InsertPosIndex, DbgValueVector &DbgValues) override
Identifiy anti-dependencies along the critical path of the ScheduleDAG and break them by renaming reg...
unsigned const MachineRegisterInfo * MRI
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineInstrBuilder & UseMI
int findRegisterDefOperandIdx(unsigned Reg, bool isDead=false, bool Overlap=false, const TargetRegisterInfo *TRI=nullptr) const
Returns the operand index that is a def of the specified register or -1 if it is not found...
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
const MachineOperand & getOperand(unsigned i) const
bool isRegTiedToUseOperand(unsigned DefOpIdx, unsigned *UseOpIdx=nullptr) const
Given the index of a register def operand, check if the register def is tied to a source operand...
void FinishBlock() override
Finish anti-dep breaking for a basic block.
MCRegAliasIterator enumerates all registers aliasing Reg.
static const unsigned End
static const SUnit * CriticalPathStep(const SUnit *SU)
CriticalPathStep - Return the next SUnit after SU on the bottom-up critical path. ...
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
bool empty() const
empty - Check if the array is empty.
succ_iterator succ_begin()
void Observe(MachineInstr &MI, unsigned Count, unsigned InsertPosIndex) override
Update liveness information to account for the current instruction, which will not be scheduled...
MCSubRegIterator enumerates all sub-registers of Reg.
bool isReturnBlock() const
Convenience function that returns true if the block ends in a return instruction. ...
bool isPredicated(const MachineInstr &MI) const override
Returns true if the instruction is already predicated.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool readsRegister(unsigned Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr reads the specified register.
MachineOperand class - Representation of each machine instruction operand.
bool test(unsigned Idx) const
std::vector< unsigned > & GetDefIndices()
Return the define indices.
bool hasExtraDefRegAllocReq(QueryType Type=AnyInBundle) const
Returns true if this instruction def operands have special register allocation requirements that are ...
void UpdateDbgValue(MachineInstr &MI, unsigned OldReg, unsigned NewReg)
Update DBG_VALUE if dependency breaker is updating other machine instruction to use NewReg...
void dump(const TargetInstrInfo *TII=nullptr) const
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
unsigned getDepth() const
getDepth - Return the depth of this node, which is the length of the maximum path up to any node whic...
bool isSuperRegister(unsigned RegA, unsigned RegB) const
Returns true if RegB is a super-register of RegA.
Representation of each machine instruction.
std::multimap< unsigned, RegisterReference > & GetRegRefs()
Return the RegRefs map.
bool isCall(QueryType Type=AnyInBundle) const
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
static cl::opt< int > DebugDiv("agg-antidep-debugdiv", cl::desc("Debug control for aggressive anti-dep breaker"), cl::init(0), cl::Hidden)
unsigned getReg() const
getReg - Returns the register number.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SmallVector< SDep, 4 > Succs
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
std::vector< std::pair< MachineInstr *, MachineInstr * > > DbgValueVector
const char * getName(unsigned RegNo) const
Return the human-readable symbolic target-specific name for the specified physical register...
Contains all the state necessary for anti-dep breaking.
static cl::opt< int > DebugMod("agg-antidep-debugmod", cl::desc("Debug control for aggressive anti-dep breaker"), cl::init(0), cl::Hidden)
unsigned getReg() const
getReg - Return the register associated with this edge.
bool IsLive(unsigned Reg)
Return true if Reg is live.
std::vector< unsigned > & GetKillIndices()
Return the kill indices.
SUnit - Scheduling unit. This is a node in the scheduling DAG.