28 #define DEBUG_TYPE "post-RA-sched"
33 TII(MF.getSubtarget().getInstrInfo()),
34 TRI(MF.getSubtarget().getRegisterInfo()), RegClassInfo(RCI),
35 Classes(TRI->getNumRegs(), nullptr), KillIndices(TRI->getNumRegs(), 0),
36 DefIndices(TRI->getNumRegs(), 0), KeepRegs(TRI->getNumRegs(),
false) {}
42 const unsigned BBSize = BB->
size();
49 DefIndices[
i] = BBSize;
60 for (
const auto &LI : (*SI)->liveins()) {
64 KillIndices[
Reg] = BBSize;
65 DefIndices[
Reg] = ~0u;
75 if (!IsReturnBlock && !Pristine.
test(*
I))
continue;
79 KillIndices[
Reg] = BBSize;
80 DefIndices[
Reg] = ~0u;
91 unsigned InsertPosIndex) {
101 assert(Count < InsertPosIndex &&
"Instruction index out of expected range!");
104 if (KillIndices[
Reg] != ~0u) {
109 KillIndices[
Reg] = Count;
110 }
else if (DefIndices[
Reg] < InsertPosIndex && DefIndices[
Reg] >= Count) {
119 DefIndices[
Reg] = InsertPosIndex;
123 PrescanInstruction(MI);
124 ScanInstruction(MI, Count);
130 const SDep *Next =
nullptr;
131 unsigned NextDepth = 0;
135 const SUnit *PredSU =
P->getSUnit();
136 unsigned PredLatency =
P->getLatency();
137 unsigned PredTotalLatency = PredSU->
getDepth() + PredLatency;
140 if (NextDepth < PredTotalLatency ||
141 (NextDepth == PredTotalLatency &&
P->getKind() ==
SDep::Anti)) {
142 NextDepth = PredTotalLatency;
149 void CriticalAntiDepBreaker::PrescanInstruction(
MachineInstr &
MI) {
173 if (!MO.
isReg())
continue;
175 if (Reg == 0)
continue;
183 if (!Classes[Reg] && NewRC)
184 Classes[
Reg] = NewRC;
185 else if (!NewRC || Classes[Reg] != NewRC)
193 unsigned AliasReg = *AI;
194 if (Classes[AliasReg]) {
201 if (Classes[Reg] != reinterpret_cast<TargetRegisterClass *>(-1))
202 RegRefs.insert(std::make_pair(Reg, &MO));
217 SubRegs.
isValid(); ++SubRegs) {
218 KeepRegs.
set(*SubRegs);
221 SuperRegs.
isValid(); ++SuperRegs) {
222 KeepRegs.
set(*SuperRegs);
226 if (MO.
isUse() && Special) {
227 if (!KeepRegs.
test(Reg)) {
230 KeepRegs.
set(*SubRegs);
236 void CriticalAntiDepBreaker::ScanInstruction(
MachineInstr &MI,
unsigned Count) {
240 assert(!MI.
isKill() &&
"Attempting to scan a kill instruction");
249 for (
unsigned i = 0, e = TRI->getNumRegs();
i != e; ++
i)
251 DefIndices[
i] = Count;
252 KillIndices[
i] = ~0u;
254 Classes[
i] =
nullptr;
258 if (!MO.
isReg())
continue;
259 unsigned Reg = MO.
getReg();
260 if (Reg == 0)
continue;
261 if (!MO.
isDef())
continue;
269 bool Keep = KeepRegs.
test(Reg);
274 unsigned SubregReg = *SRI;
275 DefIndices[SubregReg] = Count;
276 KillIndices[SubregReg] = ~0u;
277 Classes[SubregReg] =
nullptr;
278 RegRefs.erase(SubregReg);
280 KeepRegs.
reset(SubregReg);
284 Classes[*SR] = reinterpret_cast<TargetRegisterClass *>(-1);
289 if (!MO.
isReg())
continue;
290 unsigned Reg = MO.
getReg();
291 if (Reg == 0)
continue;
292 if (!MO.
isUse())
continue;
300 if (!Classes[Reg] && NewRC)
301 Classes[
Reg] = NewRC;
302 else if (!NewRC || Classes[Reg] != NewRC)
305 RegRefs.insert(std::make_pair(Reg, &MO));
310 unsigned AliasReg = *AI;
311 if (KillIndices[AliasReg] == ~0u) {
312 KillIndices[AliasReg] = Count;
313 DefIndices[AliasReg] = ~0u;
331 CriticalAntiDepBreaker::isNewRegClobberedByRefs(RegRefIter RegRefBegin,
332 RegRefIter RegRefEnd,
335 for (RegRefIter
I = RegRefBegin;
I != RegRefEnd; ++
I ) {
352 if (!CheckOper.
isReg() || !CheckOper.
isDef() ||
353 CheckOper.
getReg() != NewReg)
358 if (RefOper->
isDef())
375 unsigned CriticalAntiDepBreaker::
376 findSuitableFreeRegister(RegRefIter RegRefBegin,
377 RegRefIter RegRefEnd,
384 for (
unsigned i = 0;
i != Order.
size(); ++
i) {
385 unsigned NewReg = Order[
i];
387 if (NewReg == AntiDepReg)
continue;
391 if (NewReg == LastNewReg)
continue;
395 if (isNewRegClobberedByRefs(RegRefBegin, RegRefEnd, NewReg))
continue;
398 assert(((KillIndices[AntiDepReg] == ~0u) != (DefIndices[AntiDepReg] == ~0u))
399 &&
"Kill and Def maps aren't consistent for AntiDepReg!");
400 assert(((KillIndices[NewReg] == ~0u) != (DefIndices[NewReg] == ~0u))
401 &&
"Kill and Def maps aren't consistent for NewReg!");
402 if (KillIndices[NewReg] != ~0u ||
403 Classes[NewReg] == reinterpret_cast<TargetRegisterClass *>(-1) ||
404 KillIndices[AntiDepReg] > DefIndices[NewReg])
407 bool Forbidden =
false;
409 ite = Forbid.
end(); it != ite; ++it)
410 if (TRI->regsOverlap(NewReg, *it)) {
414 if (Forbidden)
continue;
426 unsigned InsertPosIndex,
430 if (SUnits.empty())
return 0;
440 for (
unsigned i = 0, e = SUnits.size();
i != e; ++
i) {
441 const SUnit *SU = &SUnits[
i];
449 DEBUG(
dbgs() <<
"Critical path has total latency "
452 for (
unsigned Reg = 0; Reg < TRI->getNumRegs(); ++
Reg) {
453 if (KillIndices[Reg] == ~0u)
454 DEBUG(
dbgs() <<
" " << TRI->getName(Reg));
506 std::vector<unsigned> LastNewReg(TRI->getNumRegs(), 0);
512 unsigned Count = InsertPosIndex - 1;
538 unsigned AntiDepReg = 0;
539 if (&MI == CriticalPathMI) {
541 const SUnit *NextSU = Edge->getSUnit();
545 AntiDepReg = Edge->getReg();
546 assert(AntiDepReg != 0 &&
"Anti-dependence on reg0?");
550 else if (KeepRegs.
test(AntiDepReg))
564 PE = CriticalPathSU->
Preds.end();
P != PE; ++
P)
565 if (
P->getSUnit() == NextSU ?
566 (
P->getKind() !=
SDep::Anti ||
P->getReg() != AntiDepReg) :
567 (
P->getKind() ==
SDep::Data &&
P->getReg() == AntiDepReg)) {
573 CriticalPathSU = NextSU;
574 CriticalPathMI = CriticalPathSU->
getInstr();
577 CriticalPathSU =
nullptr;
578 CriticalPathMI =
nullptr;
582 PrescanInstruction(MI);
593 else if (AntiDepReg) {
600 if (!MO.
isReg())
continue;
601 unsigned Reg = MO.
getReg();
602 if (Reg == 0)
continue;
603 if (MO.
isUse() && TRI->regsOverlap(AntiDepReg, Reg)) {
607 if (MO.
isDef() && Reg != AntiDepReg)
616 assert((AntiDepReg == 0 || RC !=
nullptr) &&
617 "Register should be live if it's causing an anti-dependence!");
618 if (RC == reinterpret_cast<TargetRegisterClass *>(-1))
625 if (AntiDepReg != 0) {
626 std::pair<std::multimap<unsigned, MachineOperand *>::iterator,
627 std::multimap<unsigned, MachineOperand *>::iterator>
628 Range = RegRefs.equal_range(AntiDepReg);
629 if (
unsigned NewReg = findSuitableFreeRegister(Range.first, Range.second,
631 LastNewReg[AntiDepReg],
633 DEBUG(
dbgs() <<
"Breaking anti-dependence edge on "
634 << TRI->getName(AntiDepReg)
635 <<
" with " << RegRefs.count(AntiDepReg) <<
" references"
636 <<
" using " << TRI->getName(NewReg) <<
"!\n");
640 for (std::multimap<unsigned, MachineOperand *>::iterator
641 Q = Range.first, QE = Range.second; Q != QE; ++Q) {
642 Q->second->setReg(NewReg);
646 const SUnit *SU = MISUnitMap[Q->second->getParent()];
648 for (DbgValueVector::iterator DVI = DbgValues.begin(),
649 DVE = DbgValues.end(); DVI != DVE; ++DVI)
650 if (DVI->second == Q->second->getParent())
657 Classes[NewReg] = Classes[AntiDepReg];
658 DefIndices[NewReg] = DefIndices[AntiDepReg];
659 KillIndices[NewReg] = KillIndices[AntiDepReg];
660 assert(((KillIndices[NewReg] == ~0u) !=
661 (DefIndices[NewReg] == ~0u)) &&
662 "Kill and Def maps aren't consistent for NewReg!");
664 Classes[AntiDepReg] =
nullptr;
665 DefIndices[AntiDepReg] = KillIndices[AntiDepReg];
666 KillIndices[AntiDepReg] = ~0u;
667 assert(((KillIndices[AntiDepReg] == ~0u) !=
668 (DefIndices[AntiDepReg] == ~0u)) &&
669 "Kill and Def maps aren't consistent for AntiDepReg!");
671 RegRefs.erase(AntiDepReg);
672 LastNewReg[AntiDepReg] = NewReg;
677 ScanInstruction(MI, Count);
void push_back(const T &Elt)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
BitVector getPristineRegs(const MachineFunction &MF) const
Return a set of physical registers that are pristine.
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
void FinishBlock() override
Finish anti-dep breaking for a basic block.
MachineInstr * getInstr() const
getInstr - Return the representative MachineInstr for this SUnit.
~CriticalAntiDepBreaker() override
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
SmallVector< SDep, 4 > Preds
A register anti-dependedence (aka WAR).
MCSuperRegIterator enumerates all super-registers of Reg.
const HexagonInstrInfo * TII
This class works in conjunction with the post-RA scheduler to rename registers to break register anti...
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Regular data dependence (aka true-dependence).
std::vector< MachineBasicBlock * >::iterator succ_iterator
Reg
All possible values of the reg field in the ModR/M byte.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
ArrayRef< MCPhysReg > getOrder(const TargetRegisterClass *RC) const
getOrder - Returns the preferred allocation order for RC.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
unsigned getNumRegs() const
Return the number of registers this target has (useful for sizing arrays holding per register informa...
bool hasExtraSrcRegAllocReq(QueryType Type=AnyInBundle) const
Returns true if this instruction source operands have special register allocation requirements that a...
Function Alias Analysis false
virtual const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const =0
Return a null-terminated list of all of the callee-saved registers on this target.
size_t size() const
size - Get the array size.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
bool isDebugValue() const
SDep - Scheduling dependency.
bool isEarlyClobber() const
unsigned const MachineRegisterInfo * MRI
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
const MachineOperand & getOperand(unsigned i) const
bool isRegTiedToUseOperand(unsigned DefOpIdx, unsigned *UseOpIdx=nullptr) const
Given the index of a register def operand, check if the register def is tied to a source operand...
MCRegAliasIterator enumerates all registers aliasing Reg.
static const unsigned End
void StartBlock(MachineBasicBlock *BB) override
Initialize anti-dep breaking for a new basic block.
CriticalAntiDepBreaker(MachineFunction &MFi, const RegisterClassInfo &)
succ_iterator succ_begin()
unsigned BreakAntiDependencies(const std::vector< SUnit > &SUnits, MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, unsigned InsertPosIndex, DbgValueVector &DbgValues) override
Identifiy anti-dependencies along the critical path of the ScheduleDAG and break them by renaming reg...
MCSubRegIterator enumerates all sub-registers of Reg.
bool isReturnBlock() const
Convenience function that returns true if the block ends in a return instruction. ...
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
MachineOperand class - Representation of each machine instruction operand.
bool test(unsigned Idx) const
bool hasExtraDefRegAllocReq(QueryType Type=AnyInBundle) const
Returns true if this instruction def operands have special register allocation requirements that are ...
void UpdateDbgValue(MachineInstr &MI, unsigned OldReg, unsigned NewReg)
Update DBG_VALUE if dependency breaker is updating other machine instruction to use NewReg...
bool isAllocatable(unsigned PhysReg) const
isAllocatable - Returns true when PhysReg belongs to an allocatable register class and it hasn't been...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static bool clobbersPhysReg(const uint32_t *RegMask, unsigned PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
unsigned getDepth() const
getDepth - Return the depth of this node, which is the length of the maximum path up to any node whic...
void Observe(MachineInstr &MI, unsigned Count, unsigned InsertPosIndex) override
Update liveness information to account for the current instruction, which will not be scheduled...
Representation of each machine instruction.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
virtual bool isPredicated(const MachineInstr &MI) const
Returns true if the instruction is already predicated.
bool isCall(QueryType Type=AnyInBundle) const
static const SDep * CriticalPathStep(const SUnit *SU)
CriticalPathStep - Return the next SUnit after SU on the bottom-up critical path. ...
unsigned getReg() const
getReg - Returns the register number.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
std::vector< std::pair< MachineInstr *, MachineInstr * > > DbgValueVector
const TargetRegisterClass * getRegClass(const MCInstrDesc &TID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum...
SUnit - Scheduling unit. This is a node in the scheduling DAG.