37 #define DEBUG_TYPE "execution-fix"
62 unsigned AvailableDomains;
74 bool isCollapsed()
const {
return Instrs.
empty(); }
77 bool hasDomain(
unsigned domain)
const {
79 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
80 "undefined behavior");
81 return AvailableDomains & (1u << domain);
85 void addDomain(
unsigned domain) {
86 AvailableDomains |= 1u << domain;
90 void setSingleDomain(
unsigned domain) {
91 AvailableDomains = 1u << domain;
95 unsigned getCommonDomains(
unsigned mask)
const {
96 return AvailableDomains & mask;
100 unsigned getFirstDomain()
const {
104 DomainValue() : Refs(0) { clear(); }
108 AvailableDomains = 0;
140 std::vector<SmallVector<int, 1>> AliasMap;
141 const unsigned NumRegs;
147 std::vector<std::pair<MachineInstr*, unsigned> > UndefReads;
158 bool SeenUnknownBackEdge;
171 const char *getPassName()
const override {
172 return "Execution dependency fix";
177 regIndices(
unsigned Reg)
const;
180 DomainValue *alloc(
int domain = -1);
181 DomainValue *retain(DomainValue *DV) {
185 void release(DomainValue*);
186 DomainValue *resolve(DomainValue*&);
189 void setLiveReg(
int rx, DomainValue *DV);
191 void force(
int rx,
unsigned domain);
192 void collapse(DomainValue *dv,
unsigned domain);
193 bool merge(DomainValue *
A, DomainValue *B);
201 bool shouldBreakDependence(
MachineInstr*,
unsigned OpIdx,
unsigned Pref);
211 ExeDepsFix::regIndices(
unsigned Reg)
const {
212 assert(Reg < AliasMap.size() &&
"Invalid register");
213 const auto &Entry = AliasMap[
Reg];
214 return make_range(Entry.begin(), Entry.end());
217 DomainValue *ExeDepsFix::alloc(
int domain) {
218 DomainValue *dv = Avail.empty() ?
219 new(Allocator.Allocate()) DomainValue :
220 Avail.pop_back_val();
222 dv->addDomain(domain);
223 assert(dv->Refs == 0 &&
"Reference count wasn't cleared");
224 assert(!dv->Next &&
"Chained DomainValue shouldn't have been recycled");
230 void ExeDepsFix::release(DomainValue *DV) {
232 assert(DV->Refs &&
"Bad DomainValue");
237 if (DV->AvailableDomains && !DV->isCollapsed())
238 collapse(DV, DV->getFirstDomain());
240 DomainValue *Next = DV->Next;
250 DomainValue *ExeDepsFix::resolve(DomainValue *&DVRef) {
251 DomainValue *DV = DVRef;
252 if (!DV || !DV->Next)
267 void ExeDepsFix::setLiveReg(
int rx, DomainValue *dv) {
268 assert(
unsigned(rx) < NumRegs &&
"Invalid index");
269 assert(LiveRegs &&
"Must enter basic block first.");
271 if (LiveRegs[rx].
Value == dv)
273 if (LiveRegs[rx].
Value)
274 release(LiveRegs[rx].Value);
275 LiveRegs[rx].Value = retain(dv);
279 void ExeDepsFix::kill(
int rx) {
280 assert(
unsigned(rx) < NumRegs &&
"Invalid index");
281 assert(LiveRegs &&
"Must enter basic block first.");
282 if (!LiveRegs[rx].
Value)
285 release(LiveRegs[rx].Value);
286 LiveRegs[rx].Value =
nullptr;
290 void ExeDepsFix::force(
int rx,
unsigned domain) {
291 assert(
unsigned(rx) < NumRegs &&
"Invalid index");
292 assert(LiveRegs &&
"Must enter basic block first.");
293 if (DomainValue *dv = LiveRegs[rx].
Value) {
294 if (dv->isCollapsed())
295 dv->addDomain(domain);
296 else if (dv->hasDomain(domain))
297 collapse(dv, domain);
301 collapse(dv, dv->getFirstDomain());
302 assert(LiveRegs[rx].Value &&
"Not live after collapse?");
303 LiveRegs[rx].Value->addDomain(domain);
307 setLiveReg(rx, alloc(domain));
313 void ExeDepsFix::collapse(DomainValue *dv,
unsigned domain) {
314 assert(dv->hasDomain(domain) &&
"Cannot collapse");
317 while (!dv->Instrs.empty())
318 TII->setExecutionDomain(dv->Instrs.pop_back_val(), domain);
319 dv->setSingleDomain(domain);
322 if (LiveRegs && dv->Refs > 1)
323 for (
unsigned rx = 0; rx != NumRegs; ++rx)
324 if (LiveRegs[rx].Value == dv)
325 setLiveReg(rx, alloc(domain));
330 assert(!A->isCollapsed() &&
"Cannot merge into collapsed");
331 assert(!B->isCollapsed() &&
"Cannot merge from collapsed");
335 unsigned common = A->getCommonDomains(B->AvailableDomains);
338 A->AvailableDomains = common;
339 A->Instrs.append(B->Instrs.begin(), B->Instrs.end());
346 for (
unsigned rx = 0; rx != NumRegs; ++rx) {
347 assert(LiveRegs &&
"no space allocated for live registers");
348 if (LiveRegs[rx].Value == B)
357 SeenUnknownBackEdge =
false;
368 LiveRegs =
new LiveReg[NumRegs];
371 for (
unsigned rx = 0; rx != NumRegs; ++rx) {
372 LiveRegs[rx].Value =
nullptr;
373 LiveRegs[rx].Def = -(1 << 20);
380 for (
int rx : regIndices(*i)) {
384 LiveRegs[rx].Def = -1;
393 pe = MBB->
pred_end(); pi != pe; ++pi) {
394 LiveOutMap::const_iterator fi = LiveOuts.find(*pi);
395 if (fi == LiveOuts.end()) {
396 SeenUnknownBackEdge =
true;
399 assert(fi->second &&
"Can't have NULL entries");
401 for (
unsigned rx = 0; rx != NumRegs; ++rx) {
403 LiveRegs[rx].Def = std::max(LiveRegs[rx].
Def, fi->second[rx].Def);
405 DomainValue *pdv = resolve(fi->second[rx].Value);
408 if (!LiveRegs[rx].Value) {
414 if (LiveRegs[rx].Value->isCollapsed()) {
416 unsigned Domain = LiveRegs[rx].Value->getFirstDomain();
417 if (!pdv->isCollapsed() && pdv->hasDomain(Domain))
418 collapse(pdv, Domain);
423 if (!pdv->isCollapsed())
424 merge(LiveRegs[rx].Value, pdv);
426 force(rx, pdv->getFirstDomain());
430 << (SeenUnknownBackEdge ?
": incomplete\n" :
": all preds known\n"));
434 assert(LiveRegs &&
"Must enter basic block first.");
437 bool First = LiveOuts.insert(std::make_pair(MBB, LiveRegs)).second;
442 for (
unsigned i = 0, e = NumRegs; i != e; ++i)
443 LiveRegs[i].
Def -= CurInstr;
447 for (
unsigned i = 0, e = NumRegs; i != e; ++i)
448 release(LiveRegs[i].Value);
459 std::pair<uint16_t, uint16_t> DomP =
TII->getExecutionDomain(MI);
462 visitSoftInstr(MI, DomP.second);
464 visitHardInstr(MI, DomP.first);
469 processDefs(MI, !DomP.first);
474 bool ExeDepsFix::shouldBreakDependence(
MachineInstr *MI,
unsigned OpIdx,
477 for (
int rx : regIndices(reg)) {
478 unsigned Clearance = CurInstr - LiveRegs[rx].Def;
479 DEBUG(
dbgs() <<
"Clearance: " << Clearance <<
", want " << Pref);
481 if (Pref > Clearance) {
482 DEBUG(
dbgs() <<
": Break dependency.\n");
487 if (!SeenUnknownBackEdge || Pref <=
unsigned(CurInstr)) {
492 DEBUG(
dbgs() <<
": Wait for back-edge to resolve.\n");
503 assert(!MI->
isDebugValue() &&
"Won't process debug values");
507 unsigned Pref =
TII->getUndefRegClearance(MI, OpNum, TRI);
509 if (shouldBreakDependence(MI, OpNum, Pref))
510 UndefReads.push_back(std::make_pair(MI, OpNum));
523 for (
int rx : regIndices(MO.
getReg())) {
525 DEBUG(
dbgs() << TRI->getName(RC->getRegister(rx)) <<
":\t" << CurInstr
530 unsigned Pref =
TII->getPartialRegUpdateClearance(MI, i, TRI);
531 if (Pref && shouldBreakDependence(MI, i, Pref))
532 TII->breakPartialRegDependency(MI, i, TRI);
535 LiveRegs[rx].Def = CurInstr;
552 if (UndefReads.empty())
560 unsigned OpIdx = UndefReads.back().second;
567 if (UndefMI == &*
I) {
569 TII->breakPartialRegDependency(UndefMI, OpIdx, TRI);
571 UndefReads.pop_back();
572 if (UndefReads.empty())
575 UndefMI = UndefReads.back().first;
576 OpIdx = UndefReads.back().second;
583 void ExeDepsFix::visitHardInstr(
MachineInstr *mi,
unsigned domain) {
588 if (!mo.
isReg())
continue;
589 for (
int rx : regIndices(mo.
getReg())) {
597 if (!mo.
isReg())
continue;
598 for (
int rx : regIndices(mo.
getReg())) {
606 void ExeDepsFix::visitSoftInstr(
MachineInstr *mi,
unsigned mask) {
609 unsigned available = mask;
617 if (!mo.
isReg())
continue;
618 for (
int rx : regIndices(mo.
getReg())) {
619 DomainValue *dv = LiveRegs[rx].Value;
623 unsigned common = dv->getCommonDomains(available);
625 if (dv->isCollapsed()) {
629 if (common) available = common;
643 TII->setExecutionDomain(mi, domain);
644 visitHardInstr(mi, domain);
653 assert(LiveRegs &&
"no space allocated for live registers");
654 const LiveReg &LR = LiveRegs[rx];
656 if (!LR.Value->getCommonDomains(available)) {
661 bool Inserted =
false;
663 i != e && !Inserted; ++i) {
664 if (LR.Def < i->Def) {
675 DomainValue *dv =
nullptr;
676 while (!Regs.
empty()) {
680 dv->AvailableDomains = dv->getCommonDomains(available);
681 assert(dv->AvailableDomains &&
"Domain should have been filtered");
687 if (Latest == dv || Latest->Next)
689 if (
merge(dv, Latest))
694 assert(LiveRegs &&
"no space allocated for live registers");
695 if (LiveRegs[i].Value == Latest)
703 dv->AvailableDomains = available;
705 dv->Instrs.push_back(mi);
713 if (!mo.
isReg())
continue;
714 for (
int rx : regIndices(mo.
getReg())) {
715 if (!LiveRegs[rx].Value || (mo.
isDef() && LiveRegs[rx].Value != dv)) {
725 TII = MF->getSubtarget().getInstrInfo();
728 assert(NumRegs == RC->getNumRegs() &&
"Bad regclass");
730 DEBUG(
dbgs() <<
"********** FIX EXECUTION DEPENDENCIES: "
731 << TRI->getRegClassName(RC) <<
" **********\n");
735 bool anyregs =
false;
738 if (MF->getRegInfo().isPhysRegUsed(*
I)) {
742 if (!anyregs)
return false;
745 if (AliasMap.empty()) {
748 AliasMap.resize(TRI->getNumRegs());
749 for (
unsigned i = 0, e = RC->getNumRegs(); i != e; ++i)
752 AliasMap[*AI].push_back(i);
759 MBBI = RPOT.begin(), MBBE = RPOT.end(); MBBI != MBBE; ++MBBI) {
761 enterBasicBlock(MBB);
762 if (SeenUnknownBackEdge)
767 processUndefReads(MBB);
768 leaveBasicBlock(MBB);
773 for (
unsigned i = 0, e = Loops.
size(); i != e; ++i) {
775 enterBasicBlock(MBB);
778 if (!
I->isDebugValue())
779 processDefs(
I,
false);
780 processUndefReads(MBB);
781 leaveBasicBlock(MBB);
786 MBBI = RPOT.begin(), MBBE = RPOT.end(); MBBI != MBBE; ++MBBI) {
787 LiveOutMap::const_iterator FI = LiveOuts.find(*MBBI);
788 if (FI == LiveOuts.end() || !FI->second)
790 for (
unsigned i = 0, e = NumRegs; i != e; ++i)
791 if (FI->second[i].Value)
792 release(FI->second[i].Value);
798 Allocator.DestroyAll();
805 return new ExeDepsFix(RC);
void push_back(const T &Elt)
const MCPhysReg * const_iterator
mop_iterator operands_end()
int getNumber() const
getNumber - MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a M...
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
std::vector< unsigned >::const_iterator livein_iterator
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
Describe properties that are true of each instruction in the target description file.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
This file defines the MallocAllocator and BumpPtrAllocator interfaces.
livein_iterator livein_begin() const
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Reg
All possible values of the reg field in the ModR/M byte.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
const HexagonRegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
reverse_iterator rbegin()
TargetInstrInfo - Interface to description of machine instruction set.
bool isDebugValue() const
bundle_iterator< MachineInstr, instr_iterator > iterator
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
livein_iterator livein_end() const
const MachineOperand & getOperand(unsigned i) const
A set of live virtual registers and physical register units.
MCRegAliasIterator enumerates all registers aliasing Reg.
Represent the analysis usage information of a pass.
bool isVariadic(QueryType Type=IgnoreBundle) const
Return true if this instruction can have a variable number of operands.
FunctionPass class - This class is used to implement most global optimizations.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
pred_iterator pred_begin()
std::vector< NodeType * >::reverse_iterator rpo_iterator
std::vector< MachineBasicBlock * >::const_iterator const_pred_iterator
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
FunctionPass * createExecutionDependencyFixPass(const TargetRegisterClass *RC)
createExecutionDependencyFixPass - This pass fixes execution time problems with dependent instruction...
A BumpPtrAllocator that allows only elements of a specific type to be allocated.
bool contains(unsigned Reg) const
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
A range adaptor for a pair of iterators.
void setPreservesAll()
Set by analyses that do not transform their input at all.
iterator insert(iterator I, T &&Elt)
Representation of each machine instruction.
A set of live physical registers with functions to track liveness when walking backward/forward throu...
unsigned getReg() const
getReg - Returns the register number.
std::reverse_iterator< iterator > reverse_iterator
LLVM Value Representation.
mop_iterator operands_begin()
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...