30 #define DEBUG_TYPE "post-RA-sched"
35 cl::desc(
"Debug control for aggressive anti-dep breaker"),
39 cl::desc(
"Debug control for aggressive anti-dep breaker"),
44 NumTargetRegs(TargetRegs), GroupNodes(TargetRegs, 0),
45 GroupNodeIndices(TargetRegs, 0),
46 KillIndices(TargetRegs, 0),
47 DefIndices(TargetRegs, 0)
49 const unsigned BBSize = BB->
size();
50 for (
unsigned i = 0; i < NumTargetRegs; ++i) {
53 GroupNodeIndices[i] = i;
56 DefIndices[i] = BBSize;
61 unsigned Node = GroupNodeIndices[
Reg];
62 while (GroupNodes[Node] != Node)
63 Node = GroupNodes[Node];
70 std::vector<unsigned> &Regs,
71 std::multimap<unsigned, AggressiveAntiDepState::RegisterReference> *RegRefs)
73 for (
unsigned Reg = 0;
Reg != NumTargetRegs; ++
Reg) {
81 assert(GroupNodes[0] == 0 &&
"GroupNode 0 not parent!");
82 assert(GroupNodeIndices[0] == 0 &&
"Reg 0 not in Group 0!");
89 unsigned Parent = (Group1 == 0) ? Group1 : Group2;
90 unsigned Other = (Parent == Group1) ? Group2 : Group1;
91 GroupNodes.at(Other) = Parent;
100 unsigned idx = GroupNodes.size();
101 GroupNodes.push_back(idx);
102 GroupNodeIndices[
Reg] = idx;
110 return((KillIndices[Reg] != ~0u) && (DefIndices[Reg] == ~0u));
117 TII(MF.getSubtarget().getInstrInfo()),
118 TRI(MF.getSubtarget().getRegisterInfo()), RegClassInfo(RCI),
122 for (
unsigned i = 0, e = CriticalPathRCs.
size(); i < e; ++i) {
124 if (CriticalPathSet.
none())
125 CriticalPathSet = CPSet;
127 CriticalPathSet |= CPSet;
130 DEBUG(
dbgs() <<
"AntiDep Critical-Path Registers:");
153 E = (*SI)->livein_end();
I != E; ++
I) {
158 DefIndices[
Reg] = ~0u;
169 if (!IsReturnBlock && !Pristine.
test(Reg))
continue;
171 unsigned AliasReg = *AI;
173 KillIndices[AliasReg] = BB->
size();
174 DefIndices[AliasReg] = ~0u;
185 unsigned InsertPosIndex) {
186 assert(Count < InsertPosIndex &&
"Instruction index out of expected range!");
188 std::set<unsigned> PassthruRegs;
189 GetPassthruRegs(MI, PassthruRegs);
190 PrescanInstruction(MI, Count, PassthruRegs);
191 ScanInstruction(MI, Count);
210 }
else if ((DefIndices[
Reg] < InsertPosIndex)
211 && (DefIndices[
Reg] >= Count)) {
212 DefIndices[
Reg] = Count;
237 void AggressiveAntiDepBreaker::GetPassthruRegs(
MachineInstr *MI,
238 std::set<unsigned>& PassthruRegs) {
241 if (!MO.
isReg())
continue;
243 IsImplicitDefUse(MI, MO)) {
244 const unsigned Reg = MO.
getReg();
247 PassthruRegs.insert(*SubRegs);
259 if (RegSet.
insert(
P->getReg()).second)
260 Edges.push_back(&*
P);
268 const SDep *Next =
nullptr;
269 unsigned NextDepth = 0;
274 const SUnit *PredSU =
P->getSUnit();
275 unsigned PredLatency =
P->getLatency();
276 unsigned PredTotalLatency = PredSU->
getDepth() + PredLatency;
279 if (NextDepth < PredTotalLatency ||
280 (NextDepth == PredTotalLatency &&
P->getKind() ==
SDep::Anti)) {
281 NextDepth = PredTotalLatency;
287 return (Next) ? Next->
getSUnit() :
nullptr;
290 void AggressiveAntiDepBreaker::HandleLastUse(
unsigned Reg,
unsigned KillIdx,
293 const char *footer) {
296 std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>&
305 DEBUG(
if (!header && footer)
dbgs() << footer);
309 if (!State->
IsLive(Reg)) {
310 KillIndices[
Reg] = KillIdx;
311 DefIndices[
Reg] = ~0u;
315 dbgs() << header << TRI->
getName(Reg); header =
nullptr; });
320 unsigned SubregReg = *SubRegs;
321 if (!State->
IsLive(SubregReg)) {
322 KillIndices[SubregReg] = KillIdx;
323 DefIndices[SubregReg] = ~0u;
324 RegRefs.erase(SubregReg);
327 dbgs() << header << TRI->
getName(Reg); header =
nullptr; });
333 DEBUG(
if (!header && footer)
dbgs() << footer);
336 void AggressiveAntiDepBreaker::PrescanInstruction(
MachineInstr *MI,
338 std::set<unsigned>& PassthruRegs) {
340 std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>&
351 unsigned Reg = MO.
getReg();
352 if (Reg == 0)
continue;
354 HandleLastUse(Reg, Count + 1,
"",
"\tDead Def: ",
"\n");
361 unsigned Reg = MO.
getReg();
362 if (Reg == 0)
continue;
378 unsigned AliasReg = *AI;
379 if (State->
IsLive(AliasReg)) {
382 TRI->
getName(AliasReg) <<
")");
391 RegRefs.insert(std::make_pair(Reg, RR));
401 unsigned Reg = MO.
getReg();
402 if (Reg == 0)
continue;
404 if (MI->
isKill() || (PassthruRegs.count(Reg) != 0))
415 if (TRI->isSuperRegister(Reg, *AI) && State->
IsLive(*AI))
418 DefIndices[*AI] = Count;
423 void AggressiveAntiDepBreaker::ScanInstruction(
MachineInstr *MI,
426 std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>&
445 bool Special = MI->
isCall() ||
454 unsigned Reg = MO.
getReg();
455 if (Reg == 0)
continue;
457 DEBUG(
dbgs() <<
" " << TRI->getName(Reg) <<
"=g" <<
463 HandleLastUse(Reg, Count,
"(last-use)");
472 if (i < MI->
getDesc().getNumOperands())
475 RegRefs.insert(std::make_pair(Reg, RR));
485 unsigned FirstReg = 0;
488 if (!MO.
isReg())
continue;
489 unsigned Reg = MO.
getReg();
490 if (Reg == 0)
continue;
493 DEBUG(
dbgs() <<
"=" << TRI->getName(Reg));
496 DEBUG(
dbgs() <<
" " << TRI->getName(Reg));
505 BitVector AggressiveAntiDepBreaker::GetRenameRegisters(
unsigned Reg) {
514 std::multimap<unsigned,
517 for (std::multimap<
unsigned,
519 QE = Range.second; Q != QE; ++Q) {
523 BitVector RCBV = TRI->getAllocatableSet(MF, RC);
531 DEBUG(
dbgs() <<
" " << TRI->getRegClassName(RC));
537 bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
538 unsigned AntiDepGroupIndex,
539 RenameOrderType& RenameOrder,
540 std::map<unsigned, unsigned> &RenameMap) {
543 std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>&
549 std::vector<unsigned> Regs;
551 assert(Regs.size() > 0 &&
"Empty register group!");
552 if (Regs.size() == 0)
558 DEBUG(
dbgs() <<
"\tRename Candidates for Group g" << AntiDepGroupIndex
560 std::map<unsigned, BitVector> RenameRegisterMap;
561 unsigned SuperReg = 0;
562 for (
unsigned i = 0, e = Regs.size(); i != e; ++i) {
563 unsigned Reg = Regs[i];
564 if ((SuperReg == 0) || TRI->isSuperRegister(SuperReg, Reg))
568 if (RegRefs.count(Reg) > 0) {
569 DEBUG(
dbgs() <<
"\t\t" << TRI->getName(Reg) <<
":");
572 RenameRegisterMap.insert(std::pair<unsigned, BitVector>(Reg, BV));
575 DEBUG(
for (
int r = BV.find_first(); r != -1; r = BV.find_next(r))
576 dbgs() <<
" " << TRI->getName(r));
582 for (
unsigned i = 0, e = Regs.size(); i != e; ++i) {
583 unsigned Reg = Regs[i];
584 if (Reg == SuperReg)
continue;
585 bool IsSub = TRI->isSubRegister(SuperReg, Reg);
596 static int renamecnt = 0;
600 dbgs() <<
"*** Performing rename " << TRI->getName(SuperReg) <<
614 TRI->getMinimalPhysRegClass(SuperReg,
MVT::Other);
618 DEBUG(
dbgs() <<
"\tEmpty Super Regclass!!\n");
624 RenameOrder.insert(RenameOrderType::value_type(SuperRC, Order.
size()));
626 unsigned OrigR = RenameOrder[SuperRC];
627 unsigned EndR = ((OrigR == Order.
size()) ? 0 : OrigR);
630 if (R == 0) R = Order.
size();
632 const unsigned NewSuperReg = Order[R];
636 if (NewSuperReg == SuperReg)
continue;
638 DEBUG(
dbgs() <<
" [" << TRI->getName(NewSuperReg) <<
':');
644 for (
unsigned i = 0, e = Regs.size(); i != e; ++i) {
645 unsigned Reg = Regs[i];
647 if (Reg == SuperReg) {
648 NewReg = NewSuperReg;
650 unsigned NewSubRegIdx = TRI->getSubRegIndex(SuperReg, Reg);
651 if (NewSubRegIdx != 0)
652 NewReg = TRI->getSubReg(NewSuperReg, NewSubRegIdx);
655 DEBUG(
dbgs() <<
" " << TRI->getName(NewReg));
659 if (!BV.test(NewReg)) {
668 if (State->
IsLive(NewReg) || (KillIndices[
Reg] > DefIndices[NewReg])) {
674 unsigned AliasReg = *AI;
675 if (State->
IsLive(AliasReg) ||
676 (KillIndices[
Reg] > DefIndices[AliasReg])) {
677 DEBUG(
dbgs() <<
"(alias " << TRI->getName(AliasReg) <<
" live)");
688 auto Range = RegRefs.equal_range(Reg);
689 for (
auto Q = Range.first, QE = Range.second; Q != QE; ++Q) {
690 auto UseMI = Q->second.Operand->getParent();
691 int Idx = UseMI->findRegisterDefOperandIdx(NewReg,
false,
true, TRI);
695 if (UseMI->getOperand(Idx).isEarlyClobber()) {
702 RenameMap.insert(std::pair<unsigned, unsigned>(Reg, NewReg));
707 RenameOrder.erase(SuperRC);
708 RenameOrder.insert(RenameOrderType::value_type(SuperRC, R));
726 const std::vector<SUnit>& SUnits,
729 unsigned InsertPosIndex,
734 std::multimap<unsigned, AggressiveAntiDepState::RegisterReference>&
739 if (SUnits.empty())
return 0;
742 RenameOrderType RenameOrder;
745 std::map<MachineInstr *, const SUnit *> MISUnitMap;
746 for (
unsigned i = 0, e = SUnits.size(); i != e; ++i) {
747 const SUnit *SU = &SUnits[i];
748 MISUnitMap.insert(std::pair<MachineInstr *, const SUnit *>(SU->
getInstr(),
755 const SUnit *CriticalPathSU =
nullptr;
757 if (CriticalPathSet.
any()) {
758 for (
unsigned i = 0, e = SUnits.size(); i != e; ++i) {
759 const SUnit *SU = &SUnits[i];
760 if (!CriticalPathSU ||
767 CriticalPathMI = CriticalPathSU->
getInstr();
771 DEBUG(
dbgs() <<
"\n===== Aggressive anti-dependency breaking\n");
773 for (
unsigned Reg = 0; Reg < TRI->getNumRegs(); ++
Reg) {
775 DEBUG(
dbgs() <<
" " << TRI->getName(Reg));
784 unsigned Count = InsertPosIndex - 1;
795 std::set<unsigned> PassthruRegs;
796 GetPassthruRegs(MI, PassthruRegs);
799 PrescanInstruction(MI, Count, PassthruRegs);
803 std::vector<const SDep *> Edges;
804 const SUnit *PathSU = MISUnitMap[
MI];
810 if (MI == CriticalPathMI) {
812 CriticalPathMI = (CriticalPathSU) ? CriticalPathSU->
getInstr() :
nullptr;
813 }
else if (CriticalPathSet.
any()) {
814 ExcludeRegs = &CriticalPathSet;
821 for (
unsigned i = 0, e = Edges.size(); i != e; ++i) {
822 const SDep *Edge = Edges[i];
828 unsigned AntiDepReg = Edge->
getReg();
829 DEBUG(
dbgs() <<
"\tAntidep reg: " << TRI->getName(AntiDepReg));
830 assert(AntiDepReg != 0 &&
"Anti-dependence on reg0?");
836 }
else if (ExcludeRegs && ExcludeRegs->
test(AntiDepReg)) {
839 DEBUG(
dbgs() <<
" (not critical-path)\n");
841 }
else if (PassthruRegs.count(AntiDepReg) != 0) {
850 assert(AntiDepOp &&
"Can't find index for defined register operand");
866 PE = PathSU->
Preds.end();
P != PE; ++
P) {
867 if (
P->getSUnit() == NextSU ?
868 (
P->getKind() !=
SDep::Anti ||
P->getReg() != AntiDepReg) :
869 (
P->getKind() ==
SDep::Data &&
P->getReg() == AntiDepReg)) {
875 PE = PathSU->
Preds.end();
P != PE; ++
P) {
876 if ((
P->getSUnit() == NextSU) && (
P->getKind() !=
SDep::Anti) &&
881 }
else if ((
P->getSUnit() != NextSU) &&
883 (
P->getReg() == AntiDepReg)) {
884 DEBUG(
dbgs() <<
" (other dependency)\n");
890 if (AntiDepReg == 0)
continue;
893 assert(AntiDepReg != 0);
894 if (AntiDepReg == 0)
continue;
897 const unsigned GroupIndex = State->
GetGroup(AntiDepReg);
898 if (GroupIndex == 0) {
906 std::map<unsigned, unsigned> RenameMap;
907 if (FindSuitableFreeRegisters(GroupIndex, RenameOrder, RenameMap)) {
908 DEBUG(
dbgs() <<
"\tBreaking anti-dependence edge on "
909 << TRI->getName(AntiDepReg) <<
":");
912 for (std::map<unsigned, unsigned>::iterator
913 S = RenameMap.begin(), E = RenameMap.end(); S != E; ++S) {
914 unsigned CurrReg = S->first;
915 unsigned NewReg = S->second;
917 DEBUG(
dbgs() <<
" " << TRI->getName(CurrReg) <<
"->" <<
918 TRI->getName(NewReg) <<
"(" <<
919 RegRefs.count(CurrReg) <<
" refs)");
923 std::pair<std::multimap<unsigned,
925 std::multimap<unsigned,
927 Range = RegRefs.equal_range(CurrReg);
928 for (std::multimap<
unsigned,
930 Q = Range.first, QE = Range.second; Q != QE; ++Q) {
935 const SUnit *SU = MISUnitMap[Q->second.Operand->getParent()];
937 for (DbgValueVector::iterator DVI = DbgValues.begin(),
938 DVE = DbgValues.end(); DVI != DVE; ++DVI)
939 if (DVI->second == Q->second.Operand->getParent())
947 RegRefs.erase(NewReg);
948 DefIndices[NewReg] = DefIndices[CurrReg];
949 KillIndices[NewReg] = KillIndices[CurrReg];
952 RegRefs.erase(CurrReg);
953 DefIndices[CurrReg] = KillIndices[CurrReg];
954 KillIndices[CurrReg] = ~0u;
955 assert(((KillIndices[CurrReg] == ~0u) !=
956 (DefIndices[CurrReg] == ~0u)) &&
957 "Kill and Def maps aren't consistent for AntiDepReg!");
966 ScanInstruction(MI, Count);
int find_first() const
find_first - Returns the index of the first set bit, -1 if none of the bits are set.
Information about a register reference within a liverange.
BitVector getPristineRegs(const MachineFunction &MF) const
Return a set of physical registers that are pristine.
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
MachineOperand * Operand
The registers operand.
MachineOperand * findRegisterDefOperand(unsigned Reg, bool isDead=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterDefOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
bool none() const
none - Returns true if none of the bits are set.
void UpdateDbgValue(MachineInstr *MI, unsigned OldReg, unsigned NewReg)
Update DBG_VALUE if dependency breaker is updating other machine instruction to use NewReg...
int find_next(unsigned Prev) const
find_next - Returns the index of the next set bit following the "Prev" bit.
std::vector< unsigned >::const_iterator livein_iterator
AggressiveAntiDepState(const unsigned TargetRegs, MachineBasicBlock *BB)
MachineInstr * getInstr() const
getInstr - Return the representative MachineInstr for this SUnit.
MachineOperand * findRegisterUseOperand(unsigned Reg, bool isKill=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool any() const
any - Returns true if any bit is set.
SmallVector< SDep, 4 > Preds
unsigned LeaveGroup(unsigned Reg)
A register anti-dependedence (aka WAR).
void StartBlock(MachineBasicBlock *BB) override
Initialize anti-dep breaking for a new basic block.
AggressiveAntiDepBreaker(MachineFunction &MFi, const RegisterClassInfo &RCI, TargetSubtargetInfo::RegClassVector &CriticalPathRCs)
unsigned GetGroup(unsigned Reg)
const HexagonInstrInfo * TII
This class works in conjunction with the post-RA scheduler to rename registers to break register anti...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Regular data dependence (aka true-dependence).
void Observe(MachineInstr *MI, unsigned Count, unsigned InsertPosIndex) override
Update liveness information to account for the current instruction, which will not be scheduled...
~AggressiveAntiDepBreaker() override
std::vector< MachineBasicBlock * >::iterator succ_iterator
Reg
All possible values of the reg field in the ModR/M byte.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
ArrayRef< MCPhysReg > getOrder(const TargetRegisterClass *RC) const
getOrder - Returns the preferred allocation order for RC.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
unsigned getNumRegs() const
Return the number of registers this target has (useful for sizing arrays holding per register informa...
A register output-dependence (aka WAW).
static void AntiDepEdges(const SUnit *SU, std::vector< const SDep * > &Edges)
AntiDepEdges - Return in Edges the anti- and output- dependencies in SU that we want to consider for ...
bool hasExtraSrcRegAllocReq(QueryType Type=AnyInBundle) const
Returns true if this instruction source operands have special register allocation requirements that a...
void GetGroupRegs(unsigned Group, std::vector< unsigned > &Regs, std::multimap< unsigned, AggressiveAntiDepState::RegisterReference > *RegRefs)
unsigned UnionGroups(unsigned Reg1, unsigned Reg2)
BitVector getAllocatableSet(const MachineFunction &MF, const TargetRegisterClass *RC=nullptr) const
getAllocatableSet - Returns a bitset indexed by register number indicating if a register is allocatab...
virtual const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const =0
getCalleeSavedRegs - Return a null-terminated list of all of the callee saved registers on this targe...
size_t size() const
size - Get the array size.
bool isDebugValue() const
SDep - Scheduling dependency.
bundle_iterator< MachineInstr, instr_iterator > iterator
initializer< Ty > init(const Ty &Val)
bool isReturn(QueryType Type=AnyInBundle) const
unsigned BreakAntiDependencies(const std::vector< SUnit > &SUnits, MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, unsigned InsertPosIndex, DbgValueVector &DbgValues) override
Identifiy anti-dependencies along the critical path of the ScheduleDAG and break them by renaming reg...
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
const MachineOperand & getOperand(unsigned i) const
bool isRegTiedToUseOperand(unsigned DefOpIdx, unsigned *UseOpIdx=nullptr) const
Given the index of a register def operand, check if the register def is tied to a source operand...
void FinishBlock() override
Finish anti-dep breaking for a basic block.
MCRegAliasIterator enumerates all registers aliasing Reg.
static const SUnit * CriticalPathStep(const SUnit *SU)
CriticalPathStep - Return the next SUnit after SU on the bottom-up critical path. ...
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
bool empty() const
empty - Check if the array is empty.
succ_iterator succ_begin()
MCSubRegIterator enumerates all sub-registers of Reg.
MachineOperand class - Representation of each machine instruction operand.
bool test(unsigned Idx) const
std::vector< unsigned > & GetDefIndices()
Return the define indices.
bool hasExtraDefRegAllocReq(QueryType Type=AnyInBundle) const
Returns true if this instruction def operands have special register allocation requirements that are ...
bool isAllocatable(unsigned PhysReg) const
isAllocatable - Returns true when PhysReg belongs to an allocatable register class and it hasn't been...
MachineFrameInfo * getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
unsigned getDepth() const
getDepth - Return the depth of this node, which is the length of the maximum path up to any node whic...
bool isSuperRegister(unsigned RegA, unsigned RegB) const
Returns true if RegB is a super-register of RegA.
Representation of each machine instruction.
std::multimap< unsigned, RegisterReference > & GetRegRefs()
Return the RegRefs map.
void setReg(unsigned Reg)
Change the register this operand corresponds to.
bool isCall(QueryType Type=AnyInBundle) const
MCInstrDesc const & getDesc(MCInstrInfo const &MCII, MCInst const &MCI)
Kind getKind() const
getKind - Return an enum value representing the kind of the dependence.
static cl::opt< int > DebugDiv("agg-antidep-debugdiv", cl::desc("Debug control for aggressive anti-dep breaker"), cl::init(0), cl::Hidden)
unsigned getReg() const
getReg - Returns the register number.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
std::vector< std::pair< MachineInstr *, MachineInstr * > > DbgValueVector
const char * getName(unsigned RegNo) const
Return the human-readable symbolic target-specific name for the specified physical register...
virtual bool isPredicated(const MachineInstr *MI) const
Returns true if the instruction is already predicated.
const TargetRegisterClass * getRegClass(const MCInstrDesc &TID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum...
Contains all the state necessary for anti-dep breaking.
static cl::opt< int > DebugMod("agg-antidep-debugmod", cl::desc("Debug control for aggressive anti-dep breaker"), cl::init(0), cl::Hidden)
unsigned getReg() const
getReg - Return the register associated with this edge.
bool IsLive(unsigned Reg)
Return true if Reg is live.
std::vector< unsigned > & GetKillIndices()
Return the kill indices.
SUnit - Scheduling unit. This is a node in the scheduling DAG.