38 #define DEBUG_TYPE "execution-fix"
63 unsigned AvailableDomains;
75 bool isCollapsed()
const {
return Instrs.
empty(); }
78 bool hasDomain(
unsigned domain)
const {
80 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
81 "undefined behavior");
82 return AvailableDomains & (1u << domain);
86 void addDomain(
unsigned domain) {
87 AvailableDomains |= 1u << domain;
91 void setSingleDomain(
unsigned domain) {
92 AvailableDomains = 1u << domain;
96 unsigned getCommonDomains(
unsigned mask)
const {
97 return AvailableDomains & mask;
101 unsigned getFirstDomain()
const {
105 DomainValue() : Refs(0) {
clear(); }
109 AvailableDomains = 0;
142 std::vector<SmallVector<int, 1>> AliasMap;
143 const unsigned NumRegs;
149 std::vector<std::pair<MachineInstr*, unsigned> > UndefReads;
160 bool SeenUnknownBackEdge;
178 StringRef getPassName()
const override {
return "Execution dependency fix"; }
182 regIndices(
unsigned Reg)
const;
185 DomainValue *alloc(
int domain = -1);
186 DomainValue *retain(DomainValue *DV) {
191 DomainValue *resolve(DomainValue*&);
194 void setLiveReg(
int rx, DomainValue *DV);
196 void force(
int rx,
unsigned domain);
197 void collapse(DomainValue *dv,
unsigned domain);
198 bool merge(DomainValue *
A, DomainValue *
B);
208 bool shouldBreakDependence(
MachineInstr*,
unsigned OpIdx,
unsigned Pref);
218 ExeDepsFix::regIndices(
unsigned Reg)
const {
219 assert(Reg < AliasMap.size() &&
"Invalid register");
220 const auto &Entry = AliasMap[
Reg];
221 return make_range(Entry.begin(), Entry.end());
224 DomainValue *ExeDepsFix::alloc(
int domain) {
225 DomainValue *dv = Avail.empty() ?
227 Avail.pop_back_val();
229 dv->addDomain(domain);
230 assert(dv->Refs == 0 &&
"Reference count wasn't cleared");
231 assert(!dv->Next &&
"Chained DomainValue shouldn't have been recycled");
239 assert(DV->Refs &&
"Bad DomainValue");
244 if (DV->AvailableDomains && !DV->isCollapsed())
245 collapse(DV, DV->getFirstDomain());
247 DomainValue *Next = DV->Next;
257 DomainValue *ExeDepsFix::resolve(DomainValue *&DVRef) {
258 DomainValue *DV = DVRef;
259 if (!DV || !DV->Next)
274 void ExeDepsFix::setLiveReg(
int rx, DomainValue *dv) {
275 assert(
unsigned(rx) < NumRegs &&
"Invalid index");
276 assert(LiveRegs &&
"Must enter basic block first.");
278 if (LiveRegs[rx].
Value == dv)
280 if (LiveRegs[rx].
Value)
282 LiveRegs[rx].Value = retain(dv);
286 void ExeDepsFix::kill(
int rx) {
287 assert(
unsigned(rx) < NumRegs &&
"Invalid index");
288 assert(LiveRegs &&
"Must enter basic block first.");
289 if (!LiveRegs[rx].
Value)
293 LiveRegs[rx].Value =
nullptr;
297 void ExeDepsFix::force(
int rx,
unsigned domain) {
298 assert(
unsigned(rx) < NumRegs &&
"Invalid index");
299 assert(LiveRegs &&
"Must enter basic block first.");
300 if (DomainValue *dv = LiveRegs[rx].
Value) {
301 if (dv->isCollapsed())
302 dv->addDomain(domain);
303 else if (dv->hasDomain(domain))
304 collapse(dv, domain);
308 collapse(dv, dv->getFirstDomain());
309 assert(LiveRegs[rx].Value &&
"Not live after collapse?");
310 LiveRegs[rx].Value->addDomain(domain);
314 setLiveReg(rx, alloc(domain));
320 void ExeDepsFix::collapse(DomainValue *dv,
unsigned domain) {
321 assert(dv->hasDomain(domain) &&
"Cannot collapse");
324 while (!dv->Instrs.empty())
325 TII->setExecutionDomain(*dv->Instrs.pop_back_val(), domain);
326 dv->setSingleDomain(domain);
329 if (LiveRegs && dv->Refs > 1)
330 for (
unsigned rx = 0; rx != NumRegs; ++rx)
331 if (LiveRegs[rx].Value == dv)
332 setLiveReg(rx, alloc(domain));
337 assert(!A->isCollapsed() &&
"Cannot merge into collapsed");
338 assert(!B->isCollapsed() &&
"Cannot merge from collapsed");
342 unsigned common = A->getCommonDomains(B->AvailableDomains);
345 A->AvailableDomains = common;
346 A->Instrs.append(B->Instrs.begin(), B->Instrs.end());
353 for (
unsigned rx = 0; rx != NumRegs; ++rx) {
354 assert(LiveRegs &&
"no space allocated for live registers");
355 if (LiveRegs[rx].Value == B)
364 SeenUnknownBackEdge =
false;
375 LiveRegs =
new LiveReg[NumRegs];
378 for (
unsigned rx = 0; rx != NumRegs; ++rx) {
379 LiveRegs[rx].Value =
nullptr;
380 LiveRegs[rx].Def = -(1 << 20);
385 for (
const auto &LI : MBB->
liveins()) {
386 for (
int rx : regIndices(LI.PhysReg)) {
390 LiveRegs[rx].Def = -1;
399 pe = MBB->
pred_end(); pi != pe; ++pi) {
400 LiveOutMap::const_iterator fi = LiveOuts.find(*pi);
401 if (fi == LiveOuts.end()) {
402 SeenUnknownBackEdge =
true;
405 assert(fi->second &&
"Can't have NULL entries");
407 for (
unsigned rx = 0; rx != NumRegs; ++rx) {
409 LiveRegs[rx].Def = std::max(LiveRegs[rx].
Def, fi->second[rx].Def);
411 DomainValue *pdv = resolve(fi->second[rx].Value);
414 if (!LiveRegs[rx].Value) {
420 if (LiveRegs[rx].Value->isCollapsed()) {
422 unsigned Domain = LiveRegs[rx].Value->getFirstDomain();
423 if (!pdv->isCollapsed() && pdv->hasDomain(Domain))
424 collapse(pdv, Domain);
429 if (!pdv->isCollapsed())
430 merge(LiveRegs[rx].Value, pdv);
432 force(rx, pdv->getFirstDomain());
436 << (SeenUnknownBackEdge ?
": incomplete\n" :
": all preds known\n"));
440 assert(LiveRegs &&
"Must enter basic block first.");
443 bool First = LiveOuts.insert(std::make_pair(MBB, LiveRegs)).second;
448 for (
unsigned i = 0, e = NumRegs;
i != e; ++
i)
449 LiveRegs[
i].
Def -= CurInstr;
453 for (
unsigned i = 0, e = NumRegs;
i != e; ++
i)
465 std::pair<uint16_t, uint16_t> DomP =
TII->getExecutionDomain(*MI);
468 visitSoftInstr(MI, DomP.second);
470 visitHardInstr(MI, DomP.first);
475 processDefs(MI, !DomP.first);
481 void ExeDepsFix::pickBestRegisterForUndef(
MachineInstr *MI,
unsigned OpIdx,
486 unsigned OriginalReg = MO.
getReg();
489 if (AliasMap[OriginalReg].size() != 1)
494 TII->getRegClass(MI->
getDesc(), OpIdx, TRI, *MF);
499 if (!CurrMO.isReg() || CurrMO.isDef() || CurrMO.isUndef() ||
504 MO.
setReg(CurrMO.getReg());
510 unsigned MaxClearance = 0;
511 unsigned MaxClearanceReg = OriginalReg;
513 for (
auto Reg : Order) {
514 assert(AliasMap[Reg].size() == 1 &&
515 "Reg is expected to be mapped to a single index");
516 int RCrx = *regIndices(Reg).begin();
517 unsigned Clearance = CurInstr - LiveRegs[RCrx].Def;
518 if (Clearance <= MaxClearance)
520 MaxClearance = Clearance;
521 MaxClearanceReg =
Reg;
523 if (MaxClearance > Pref)
528 if (MaxClearanceReg != OriginalReg)
529 MO.
setReg(MaxClearanceReg);
534 bool ExeDepsFix::shouldBreakDependence(
MachineInstr *MI,
unsigned OpIdx,
537 for (
int rx : regIndices(reg)) {
538 unsigned Clearance = CurInstr - LiveRegs[rx].Def;
539 DEBUG(
dbgs() <<
"Clearance: " << Clearance <<
", want " << Pref);
541 if (Pref > Clearance) {
542 DEBUG(
dbgs() <<
": Break dependency.\n");
547 if (!SeenUnknownBackEdge || Pref <=
unsigned(CurInstr)) {
552 DEBUG(
dbgs() <<
": Wait for back-edge to resolve.\n");
567 unsigned Pref =
TII->getUndefRegClearance(*MI, OpNum, TRI);
569 pickBestRegisterForUndef(MI, OpNum, Pref);
570 if (shouldBreakDependence(MI, OpNum, Pref))
571 UndefReads.push_back(std::make_pair(MI, OpNum));
582 for (
int rx : regIndices(MO.
getReg())) {
584 DEBUG(
dbgs() << TRI->getName(RC->getRegister(rx)) <<
":\t" << CurInstr
589 unsigned Pref =
TII->getPartialRegUpdateClearance(*MI,
i, TRI);
590 if (Pref && shouldBreakDependence(MI,
i, Pref))
591 TII->breakPartialRegDependency(*MI,
i, TRI);
594 LiveRegs[rx].Def = CurInstr;
611 if (UndefReads.empty())
621 unsigned OpIdx = UndefReads.back().second;
629 TII->breakPartialRegDependency(*UndefMI, OpIdx, TRI);
631 UndefReads.pop_back();
632 if (UndefReads.empty())
635 UndefMI = UndefReads.back().first;
636 OpIdx = UndefReads.back().second;
643 void ExeDepsFix::visitHardInstr(
MachineInstr *mi,
unsigned domain) {
648 if (!mo.
isReg())
continue;
649 for (
int rx : regIndices(mo.
getReg())) {
657 if (!mo.
isReg())
continue;
658 for (
int rx : regIndices(mo.
getReg())) {
666 void ExeDepsFix::visitSoftInstr(
MachineInstr *mi,
unsigned mask) {
669 unsigned available = mask;
677 if (!mo.
isReg())
continue;
678 for (
int rx : regIndices(mo.
getReg())) {
679 DomainValue *dv = LiveRegs[rx].Value;
683 unsigned common = dv->getCommonDomains(available);
685 if (dv->isCollapsed()) {
689 if (common) available = common;
703 TII->setExecutionDomain(*mi, domain);
704 visitHardInstr(mi, domain);
711 for (
int rx : used) {
712 assert(LiveRegs &&
"no space allocated for live registers");
713 const LiveReg &LR = LiveRegs[rx];
715 if (!LR.Value->getCommonDomains(available)) {
720 auto I = std::upper_bound(Regs.
begin(), Regs.
end(), &LR,
721 [](
const LiveReg *LHS,
const LiveReg *RHS) {
722 return LHS->Def < RHS->Def;
729 DomainValue *dv =
nullptr;
730 while (!Regs.
empty()) {
734 dv->AvailableDomains = dv->getCommonDomains(available);
735 assert(dv->AvailableDomains &&
"Domain should have been filtered");
741 if (Latest == dv || Latest->Next)
743 if (
merge(dv, Latest))
748 assert(LiveRegs &&
"no space allocated for live registers");
749 if (LiveRegs[
i].Value == Latest)
757 dv->AvailableDomains = available;
759 dv->Instrs.push_back(mi);
767 if (!mo.
isReg())
continue;
768 for (
int rx : regIndices(mo.
getReg())) {
769 if (!LiveRegs[rx].Value || (mo.
isDef() && LiveRegs[rx].Value != dv)) {
781 TII = MF->getSubtarget().getInstrInfo();
783 RegClassInfo.runOnMachineFunction(mf);
785 assert(NumRegs == RC->getNumRegs() &&
"Bad regclass");
787 DEBUG(
dbgs() <<
"********** FIX EXECUTION DEPENDENCIES: "
788 << TRI->getRegClassName(RC) <<
" **********\n");
792 bool anyregs =
false;
794 for (
unsigned Reg : *RC) {
800 if (!anyregs)
return false;
803 if (AliasMap.empty()) {
806 AliasMap.resize(TRI->getNumRegs());
807 for (
unsigned i = 0, e = RC->getNumRegs();
i != e; ++
i)
810 AliasMap[*AI].push_back(
i);
817 MBBI = RPOT.begin(), MBBE = RPOT.end(); MBBI != MBBE; ++MBBI) {
819 enterBasicBlock(MBB);
820 if (SeenUnknownBackEdge)
824 processUndefReads(MBB);
825 leaveBasicBlock(MBB);
831 enterBasicBlock(MBB);
834 processDefs(&MI,
false);
835 processUndefReads(MBB);
836 leaveBasicBlock(MBB);
841 MBBI = RPOT.begin(), MBBE = RPOT.end(); MBBI != MBBE; ++MBBI) {
842 LiveOutMap::const_iterator FI = LiveOuts.find(*MBBI);
843 if (FI == LiveOuts.end() || !FI->second)
845 for (
unsigned i = 0, e = NumRegs;
i != e; ++
i)
846 if (FI->second[
i].Value)
860 return new ExeDepsFix(RC);
void push_back(const T &Elt)
mop_iterator operands_end()
iterator_range< livein_iterator > liveins() const
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
Describe properties that are true of each instruction in the target description file.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
iterator_range< mop_iterator > operands()
This file defines the MallocAllocator and BumpPtrAllocator interfaces.
bool isPhysRegUsed(unsigned PhysReg) const
Return true if the specified register is modified or read in this function.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Reg
All possible values of the reg field in the ModR/M byte.
LLVM_NODISCARD bool empty() const
unsigned getNumOperands() const
Access to explicit operands of the instruction.
const HexagonRegisterInfo & getRegisterInfo() const
HexagonInstrInfo specifics.
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
reverse_iterator rbegin()
TargetInstrInfo - Interface to description of machine instruction set.
bool isDebugValue() const
void init(const MachineRegisterInfo &MRI)
unsigned const MachineRegisterInfo * MRI
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
constexpr bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
const MachineOperand & getOperand(unsigned i) const
MCRegAliasIterator enumerates all registers aliasing Reg.
Represent the analysis usage information of a pass.
Greedy Register Allocator
std::vector< NodeRef >::reverse_iterator rpo_iterator
bool isVariadic(QueryType Type=IgnoreBundle) const
Return true if this instruction can have a variable number of operands.
FunctionPass class - This class is used to implement most global optimizations.
A set of live virtual registers and physical register units.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
pred_iterator pred_begin()
std::vector< MachineBasicBlock * >::const_iterator const_pred_iterator
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
FunctionPass * createExecutionDependencyFixPass(const TargetRegisterClass *RC)
createExecutionDependencyFixPass - This pass fixes execution time problems with dependent instruction...
LLVM_NODISCARD T pop_back_val()
A BumpPtrAllocator that allows only elements of a specific type to be allocated.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
A range adaptor for a pair of iterators.
void setPreservesAll()
Set by analyses that do not transform their input at all.
static void clear(coro::Shape &Shape)
iterator insert(iterator I, T &&Elt)
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
MachineFunctionProperties & set(Property P)
LaneBitmask contains(unsigned Reg) const
Representation of each machine instruction.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
A set of live physical registers with functions to track liveness when walking backward/forward throu...
void setReg(unsigned Reg)
Change the register this operand corresponds to.
unsigned getReg() const
getReg - Returns the register number.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
mop_iterator operands_begin()
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
StringRef - Represent a constant reference to a string, i.e.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
Properties which a MachineFunction may have at a given point in time.
bool contains(unsigned Reg) const
Return true if the specified register is included in this register class.