LLVM  3.7.0
ExecutionDepsFix.cpp
Go to the documentation of this file.
1 //===- ExecutionDepsFix.cpp - Fix execution dependecy issues ----*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains the execution dependency fix pass.
11 //
12 // Some X86 SSE instructions like mov, and, or, xor are available in different
13 // variants for different operand types. These variant instructions are
14 // equivalent, but on Nehalem and newer cpus there is extra latency
15 // transferring data between integer and floating point domains. ARM cores
16 // have similar issues when they are configured with both VFP and NEON
17 // pipelines.
18 //
19 // This pass changes the variant instructions to minimize domain crossings.
20 //
21 //===----------------------------------------------------------------------===//
22 
23 #include "llvm/CodeGen/Passes.h"
29 #include "llvm/Support/Allocator.h"
30 #include "llvm/Support/Debug.h"
34 
35 using namespace llvm;
36 
37 #define DEBUG_TYPE "execution-fix"
38 
39 /// A DomainValue is a bit like LiveIntervals' ValNo, but it also keeps track
40 /// of execution domains.
41 ///
42 /// An open DomainValue represents a set of instructions that can still switch
43 /// execution domain. Multiple registers may refer to the same open
44 /// DomainValue - they will eventually be collapsed to the same execution
45 /// domain.
46 ///
47 /// A collapsed DomainValue represents a single register that has been forced
48 /// into one of more execution domains. There is a separate collapsed
49 /// DomainValue for each register, but it may contain multiple execution
50 /// domains. A register value is initially created in a single execution
51 /// domain, but if we were forced to pay the penalty of a domain crossing, we
52 /// keep track of the fact that the register is now available in multiple
53 /// domains.
54 namespace {
55 struct DomainValue {
56  // Basic reference counting.
57  unsigned Refs;
58 
59  // Bitmask of available domains. For an open DomainValue, it is the still
60  // possible domains for collapsing. For a collapsed DomainValue it is the
61  // domains where the register is available for free.
62  unsigned AvailableDomains;
63 
64  // Pointer to the next DomainValue in a chain. When two DomainValues are
65  // merged, Victim.Next is set to point to Victor, so old DomainValue
66  // references can be updated by following the chain.
67  DomainValue *Next;
68 
69  // Twiddleable instructions using or defining these registers.
71 
72  // A collapsed DomainValue has no instructions to twiddle - it simply keeps
73  // track of the domains where the registers are already available.
74  bool isCollapsed() const { return Instrs.empty(); }
75 
76  // Is domain available?
77  bool hasDomain(unsigned domain) const {
78  assert(domain <
79  static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
80  "undefined behavior");
81  return AvailableDomains & (1u << domain);
82  }
83 
84  // Mark domain as available.
85  void addDomain(unsigned domain) {
86  AvailableDomains |= 1u << domain;
87  }
88 
89  // Restrict to a single domain available.
90  void setSingleDomain(unsigned domain) {
91  AvailableDomains = 1u << domain;
92  }
93 
94  // Return bitmask of domains that are available and in mask.
95  unsigned getCommonDomains(unsigned mask) const {
96  return AvailableDomains & mask;
97  }
98 
99  // First domain available.
100  unsigned getFirstDomain() const {
101  return countTrailingZeros(AvailableDomains);
102  }
103 
104  DomainValue() : Refs(0) { clear(); }
105 
106  // Clear this DomainValue and point to next which has all its data.
107  void clear() {
108  AvailableDomains = 0;
109  Next = nullptr;
110  Instrs.clear();
111  }
112 };
113 }
114 
115 namespace {
116 /// Information about a live register.
117 struct LiveReg {
118  /// Value currently in this register, or NULL when no value is being tracked.
119  /// This counts as a DomainValue reference.
120  DomainValue *Value;
121 
122  /// Instruction that defined this register, relative to the beginning of the
123  /// current basic block. When a LiveReg is used to represent a live-out
124  /// register, this value is relative to the end of the basic block, so it
125  /// will be a negative number.
126  int Def;
127 };
128 } // anonymous namespace
129 
130 namespace {
131 class ExeDepsFix : public MachineFunctionPass {
132  static char ID;
135 
136  const TargetRegisterClass *const RC;
137  MachineFunction *MF;
138  const TargetInstrInfo *TII;
139  const TargetRegisterInfo *TRI;
140  std::vector<SmallVector<int, 1>> AliasMap;
141  const unsigned NumRegs;
142  LiveReg *LiveRegs;
143  typedef DenseMap<MachineBasicBlock*, LiveReg*> LiveOutMap;
144  LiveOutMap LiveOuts;
145 
146  /// List of undefined register reads in this block in forward order.
147  std::vector<std::pair<MachineInstr*, unsigned> > UndefReads;
148 
149  /// Storage for register unit liveness.
151 
152  /// Current instruction number.
153  /// The first instruction in each basic block is 0.
154  int CurInstr;
155 
156  /// True when the current block has a predecessor that hasn't been visited
157  /// yet.
158  bool SeenUnknownBackEdge;
159 
160 public:
161  ExeDepsFix(const TargetRegisterClass *rc)
162  : MachineFunctionPass(ID), RC(rc), NumRegs(RC->getNumRegs()) {}
163 
164  void getAnalysisUsage(AnalysisUsage &AU) const override {
165  AU.setPreservesAll();
167  }
168 
169  bool runOnMachineFunction(MachineFunction &MF) override;
170 
171  const char *getPassName() const override {
172  return "Execution dependency fix";
173  }
174 
175 private:
177  regIndices(unsigned Reg) const;
178 
179  // DomainValue allocation.
180  DomainValue *alloc(int domain = -1);
181  DomainValue *retain(DomainValue *DV) {
182  if (DV) ++DV->Refs;
183  return DV;
184  }
185  void release(DomainValue*);
186  DomainValue *resolve(DomainValue*&);
187 
188  // LiveRegs manipulations.
189  void setLiveReg(int rx, DomainValue *DV);
190  void kill(int rx);
191  void force(int rx, unsigned domain);
192  void collapse(DomainValue *dv, unsigned domain);
193  bool merge(DomainValue *A, DomainValue *B);
194 
195  void enterBasicBlock(MachineBasicBlock*);
196  void leaveBasicBlock(MachineBasicBlock*);
197  void visitInstr(MachineInstr*);
198  void processDefs(MachineInstr*, bool Kill);
199  void visitSoftInstr(MachineInstr*, unsigned mask);
200  void visitHardInstr(MachineInstr*, unsigned domain);
201  bool shouldBreakDependence(MachineInstr*, unsigned OpIdx, unsigned Pref);
202  void processUndefReads(MachineBasicBlock*);
203 };
204 }
205 
206 char ExeDepsFix::ID = 0;
207 
208 /// Translate TRI register number to a list of indices into our smaller tables
209 /// of interesting registers.
211 ExeDepsFix::regIndices(unsigned Reg) const {
212  assert(Reg < AliasMap.size() && "Invalid register");
213  const auto &Entry = AliasMap[Reg];
214  return make_range(Entry.begin(), Entry.end());
215 }
216 
217 DomainValue *ExeDepsFix::alloc(int domain) {
218  DomainValue *dv = Avail.empty() ?
219  new(Allocator.Allocate()) DomainValue :
220  Avail.pop_back_val();
221  if (domain >= 0)
222  dv->addDomain(domain);
223  assert(dv->Refs == 0 && "Reference count wasn't cleared");
224  assert(!dv->Next && "Chained DomainValue shouldn't have been recycled");
225  return dv;
226 }
227 
228 /// Release a reference to DV. When the last reference is released,
229 /// collapse if needed.
230 void ExeDepsFix::release(DomainValue *DV) {
231  while (DV) {
232  assert(DV->Refs && "Bad DomainValue");
233  if (--DV->Refs)
234  return;
235 
236  // There are no more DV references. Collapse any contained instructions.
237  if (DV->AvailableDomains && !DV->isCollapsed())
238  collapse(DV, DV->getFirstDomain());
239 
240  DomainValue *Next = DV->Next;
241  DV->clear();
242  Avail.push_back(DV);
243  // Also release the next DomainValue in the chain.
244  DV = Next;
245  }
246 }
247 
248 /// Follow the chain of dead DomainValues until a live DomainValue is reached.
249 /// Update the referenced pointer when necessary.
250 DomainValue *ExeDepsFix::resolve(DomainValue *&DVRef) {
251  DomainValue *DV = DVRef;
252  if (!DV || !DV->Next)
253  return DV;
254 
255  // DV has a chain. Find the end.
256  do DV = DV->Next;
257  while (DV->Next);
258 
259  // Update DVRef to point to DV.
260  retain(DV);
261  release(DVRef);
262  DVRef = DV;
263  return DV;
264 }
265 
266 /// Set LiveRegs[rx] = dv, updating reference counts.
267 void ExeDepsFix::setLiveReg(int rx, DomainValue *dv) {
268  assert(unsigned(rx) < NumRegs && "Invalid index");
269  assert(LiveRegs && "Must enter basic block first.");
270 
271  if (LiveRegs[rx].Value == dv)
272  return;
273  if (LiveRegs[rx].Value)
274  release(LiveRegs[rx].Value);
275  LiveRegs[rx].Value = retain(dv);
276 }
277 
278 // Kill register rx, recycle or collapse any DomainValue.
279 void ExeDepsFix::kill(int rx) {
280  assert(unsigned(rx) < NumRegs && "Invalid index");
281  assert(LiveRegs && "Must enter basic block first.");
282  if (!LiveRegs[rx].Value)
283  return;
284 
285  release(LiveRegs[rx].Value);
286  LiveRegs[rx].Value = nullptr;
287 }
288 
289 /// Force register rx into domain.
290 void ExeDepsFix::force(int rx, unsigned domain) {
291  assert(unsigned(rx) < NumRegs && "Invalid index");
292  assert(LiveRegs && "Must enter basic block first.");
293  if (DomainValue *dv = LiveRegs[rx].Value) {
294  if (dv->isCollapsed())
295  dv->addDomain(domain);
296  else if (dv->hasDomain(domain))
297  collapse(dv, domain);
298  else {
299  // This is an incompatible open DomainValue. Collapse it to whatever and
300  // force the new value into domain. This costs a domain crossing.
301  collapse(dv, dv->getFirstDomain());
302  assert(LiveRegs[rx].Value && "Not live after collapse?");
303  LiveRegs[rx].Value->addDomain(domain);
304  }
305  } else {
306  // Set up basic collapsed DomainValue.
307  setLiveReg(rx, alloc(domain));
308  }
309 }
310 
311 /// Collapse open DomainValue into given domain. If there are multiple
312 /// registers using dv, they each get a unique collapsed DomainValue.
313 void ExeDepsFix::collapse(DomainValue *dv, unsigned domain) {
314  assert(dv->hasDomain(domain) && "Cannot collapse");
315 
316  // Collapse all the instructions.
317  while (!dv->Instrs.empty())
318  TII->setExecutionDomain(dv->Instrs.pop_back_val(), domain);
319  dv->setSingleDomain(domain);
320 
321  // If there are multiple users, give them new, unique DomainValues.
322  if (LiveRegs && dv->Refs > 1)
323  for (unsigned rx = 0; rx != NumRegs; ++rx)
324  if (LiveRegs[rx].Value == dv)
325  setLiveReg(rx, alloc(domain));
326 }
327 
328 /// All instructions and registers in B are moved to A, and B is released.
329 bool ExeDepsFix::merge(DomainValue *A, DomainValue *B) {
330  assert(!A->isCollapsed() && "Cannot merge into collapsed");
331  assert(!B->isCollapsed() && "Cannot merge from collapsed");
332  if (A == B)
333  return true;
334  // Restrict to the domains that A and B have in common.
335  unsigned common = A->getCommonDomains(B->AvailableDomains);
336  if (!common)
337  return false;
338  A->AvailableDomains = common;
339  A->Instrs.append(B->Instrs.begin(), B->Instrs.end());
340 
341  // Clear the old DomainValue so we won't try to swizzle instructions twice.
342  B->clear();
343  // All uses of B are referred to A.
344  B->Next = retain(A);
345 
346  for (unsigned rx = 0; rx != NumRegs; ++rx) {
347  assert(LiveRegs && "no space allocated for live registers");
348  if (LiveRegs[rx].Value == B)
349  setLiveReg(rx, A);
350  }
351  return true;
352 }
353 
354 /// Set up LiveRegs by merging predecessor live-out values.
355 void ExeDepsFix::enterBasicBlock(MachineBasicBlock *MBB) {
356  // Detect back-edges from predecessors we haven't processed yet.
357  SeenUnknownBackEdge = false;
358 
359  // Reset instruction counter in each basic block.
360  CurInstr = 0;
361 
362  // Set up UndefReads to track undefined register reads.
363  UndefReads.clear();
364  LiveRegSet.clear();
365 
366  // Set up LiveRegs to represent registers entering MBB.
367  if (!LiveRegs)
368  LiveRegs = new LiveReg[NumRegs];
369 
370  // Default values are 'nothing happened a long time ago'.
371  for (unsigned rx = 0; rx != NumRegs; ++rx) {
372  LiveRegs[rx].Value = nullptr;
373  LiveRegs[rx].Def = -(1 << 20);
374  }
375 
376  // This is the entry block.
377  if (MBB->pred_empty()) {
379  e = MBB->livein_end(); i != e; ++i) {
380  for (int rx : regIndices(*i)) {
381  // Treat function live-ins as if they were defined just before the first
382  // instruction. Usually, function arguments are set up immediately
383  // before the call.
384  LiveRegs[rx].Def = -1;
385  }
386  }
387  DEBUG(dbgs() << "BB#" << MBB->getNumber() << ": entry\n");
388  return;
389  }
390 
391  // Try to coalesce live-out registers from predecessors.
393  pe = MBB->pred_end(); pi != pe; ++pi) {
394  LiveOutMap::const_iterator fi = LiveOuts.find(*pi);
395  if (fi == LiveOuts.end()) {
396  SeenUnknownBackEdge = true;
397  continue;
398  }
399  assert(fi->second && "Can't have NULL entries");
400 
401  for (unsigned rx = 0; rx != NumRegs; ++rx) {
402  // Use the most recent predecessor def for each register.
403  LiveRegs[rx].Def = std::max(LiveRegs[rx].Def, fi->second[rx].Def);
404 
405  DomainValue *pdv = resolve(fi->second[rx].Value);
406  if (!pdv)
407  continue;
408  if (!LiveRegs[rx].Value) {
409  setLiveReg(rx, pdv);
410  continue;
411  }
412 
413  // We have a live DomainValue from more than one predecessor.
414  if (LiveRegs[rx].Value->isCollapsed()) {
415  // We are already collapsed, but predecessor is not. Force it.
416  unsigned Domain = LiveRegs[rx].Value->getFirstDomain();
417  if (!pdv->isCollapsed() && pdv->hasDomain(Domain))
418  collapse(pdv, Domain);
419  continue;
420  }
421 
422  // Currently open, merge in predecessor.
423  if (!pdv->isCollapsed())
424  merge(LiveRegs[rx].Value, pdv);
425  else
426  force(rx, pdv->getFirstDomain());
427  }
428  }
429  DEBUG(dbgs() << "BB#" << MBB->getNumber()
430  << (SeenUnknownBackEdge ? ": incomplete\n" : ": all preds known\n"));
431 }
432 
433 void ExeDepsFix::leaveBasicBlock(MachineBasicBlock *MBB) {
434  assert(LiveRegs && "Must enter basic block first.");
435  // Save live registers at end of MBB - used by enterBasicBlock().
436  // Also use LiveOuts as a visited set to detect back-edges.
437  bool First = LiveOuts.insert(std::make_pair(MBB, LiveRegs)).second;
438 
439  if (First) {
440  // LiveRegs was inserted in LiveOuts. Adjust all defs to be relative to
441  // the end of this block instead of the beginning.
442  for (unsigned i = 0, e = NumRegs; i != e; ++i)
443  LiveRegs[i].Def -= CurInstr;
444  } else {
445  // Insertion failed, this must be the second pass.
446  // Release all the DomainValues instead of keeping them.
447  for (unsigned i = 0, e = NumRegs; i != e; ++i)
448  release(LiveRegs[i].Value);
449  delete[] LiveRegs;
450  }
451  LiveRegs = nullptr;
452 }
453 
454 void ExeDepsFix::visitInstr(MachineInstr *MI) {
455  if (MI->isDebugValue())
456  return;
457 
458  // Update instructions with explicit execution domains.
459  std::pair<uint16_t, uint16_t> DomP = TII->getExecutionDomain(MI);
460  if (DomP.first) {
461  if (DomP.second)
462  visitSoftInstr(MI, DomP.second);
463  else
464  visitHardInstr(MI, DomP.first);
465  }
466 
467  // Process defs to track register ages, and kill values clobbered by generic
468  // instructions.
469  processDefs(MI, !DomP.first);
470 }
471 
472 /// \brief Return true to if it makes sense to break dependence on a partial def
473 /// or undef use.
474 bool ExeDepsFix::shouldBreakDependence(MachineInstr *MI, unsigned OpIdx,
475  unsigned Pref) {
476  unsigned reg = MI->getOperand(OpIdx).getReg();
477  for (int rx : regIndices(reg)) {
478  unsigned Clearance = CurInstr - LiveRegs[rx].Def;
479  DEBUG(dbgs() << "Clearance: " << Clearance << ", want " << Pref);
480 
481  if (Pref > Clearance) {
482  DEBUG(dbgs() << ": Break dependency.\n");
483  continue;
484  }
485  // The current clearance seems OK, but we may be ignoring a def from a
486  // back-edge.
487  if (!SeenUnknownBackEdge || Pref <= unsigned(CurInstr)) {
488  DEBUG(dbgs() << ": OK .\n");
489  return false;
490  }
491  // A def from an unprocessed back-edge may make us break this dependency.
492  DEBUG(dbgs() << ": Wait for back-edge to resolve.\n");
493  return false;
494  }
495  return true;
496 }
497 
498 // Update def-ages for registers defined by MI.
499 // If Kill is set, also kill off DomainValues clobbered by the defs.
500 //
501 // Also break dependencies on partial defs and undef uses.
502 void ExeDepsFix::processDefs(MachineInstr *MI, bool Kill) {
503  assert(!MI->isDebugValue() && "Won't process debug values");
504 
505  // Break dependence on undef uses. Do this before updating LiveRegs below.
506  unsigned OpNum;
507  unsigned Pref = TII->getUndefRegClearance(MI, OpNum, TRI);
508  if (Pref) {
509  if (shouldBreakDependence(MI, OpNum, Pref))
510  UndefReads.push_back(std::make_pair(MI, OpNum));
511  }
512  const MCInstrDesc &MCID = MI->getDesc();
513  for (unsigned i = 0,
514  e = MI->isVariadic() ? MI->getNumOperands() : MCID.getNumDefs();
515  i != e; ++i) {
516  MachineOperand &MO = MI->getOperand(i);
517  if (!MO.isReg())
518  continue;
519  if (MO.isImplicit())
520  break;
521  if (MO.isUse())
522  continue;
523  for (int rx : regIndices(MO.getReg())) {
524  // This instruction explicitly defines rx.
525  DEBUG(dbgs() << TRI->getName(RC->getRegister(rx)) << ":\t" << CurInstr
526  << '\t' << *MI);
527 
528  // Check clearance before partial register updates.
529  // Call breakDependence before setting LiveRegs[rx].Def.
530  unsigned Pref = TII->getPartialRegUpdateClearance(MI, i, TRI);
531  if (Pref && shouldBreakDependence(MI, i, Pref))
532  TII->breakPartialRegDependency(MI, i, TRI);
533 
534  // How many instructions since rx was last written?
535  LiveRegs[rx].Def = CurInstr;
536 
537  // Kill off domains redefined by generic instructions.
538  if (Kill)
539  kill(rx);
540  }
541  }
542  ++CurInstr;
543 }
544 
545 /// \break Break false dependencies on undefined register reads.
546 ///
547 /// Walk the block backward computing precise liveness. This is expensive, so we
548 /// only do it on demand. Note that the occurrence of undefined register reads
549 /// that should be broken is very rare, but when they occur we may have many in
550 /// a single block.
551 void ExeDepsFix::processUndefReads(MachineBasicBlock *MBB) {
552  if (UndefReads.empty())
553  return;
554 
555  // Collect this block's live out register units.
556  LiveRegSet.init(TRI);
557  LiveRegSet.addLiveOuts(MBB);
558 
559  MachineInstr *UndefMI = UndefReads.back().first;
560  unsigned OpIdx = UndefReads.back().second;
561 
562  for (MachineBasicBlock::reverse_iterator I = MBB->rbegin(), E = MBB->rend();
563  I != E; ++I) {
564  // Update liveness, including the current instruction's defs.
565  LiveRegSet.stepBackward(*I);
566 
567  if (UndefMI == &*I) {
568  if (!LiveRegSet.contains(UndefMI->getOperand(OpIdx).getReg()))
569  TII->breakPartialRegDependency(UndefMI, OpIdx, TRI);
570 
571  UndefReads.pop_back();
572  if (UndefReads.empty())
573  return;
574 
575  UndefMI = UndefReads.back().first;
576  OpIdx = UndefReads.back().second;
577  }
578  }
579 }
580 
581 // A hard instruction only works in one domain. All input registers will be
582 // forced into that domain.
583 void ExeDepsFix::visitHardInstr(MachineInstr *mi, unsigned domain) {
584  // Collapse all uses.
585  for (unsigned i = mi->getDesc().getNumDefs(),
586  e = mi->getDesc().getNumOperands(); i != e; ++i) {
587  MachineOperand &mo = mi->getOperand(i);
588  if (!mo.isReg()) continue;
589  for (int rx : regIndices(mo.getReg())) {
590  force(rx, domain);
591  }
592  }
593 
594  // Kill all defs and force them.
595  for (unsigned i = 0, e = mi->getDesc().getNumDefs(); i != e; ++i) {
596  MachineOperand &mo = mi->getOperand(i);
597  if (!mo.isReg()) continue;
598  for (int rx : regIndices(mo.getReg())) {
599  kill(rx);
600  force(rx, domain);
601  }
602  }
603 }
604 
605 // A soft instruction can be changed to work in other domains given by mask.
606 void ExeDepsFix::visitSoftInstr(MachineInstr *mi, unsigned mask) {
607  // Bitmask of available domains for this instruction after taking collapsed
608  // operands into account.
609  unsigned available = mask;
610 
611  // Scan the explicit use operands for incoming domains.
612  SmallVector<int, 4> used;
613  if (LiveRegs)
614  for (unsigned i = mi->getDesc().getNumDefs(),
615  e = mi->getDesc().getNumOperands(); i != e; ++i) {
616  MachineOperand &mo = mi->getOperand(i);
617  if (!mo.isReg()) continue;
618  for (int rx : regIndices(mo.getReg())) {
619  DomainValue *dv = LiveRegs[rx].Value;
620  if (dv == nullptr)
621  continue;
622  // Bitmask of domains that dv and available have in common.
623  unsigned common = dv->getCommonDomains(available);
624  // Is it possible to use this collapsed register for free?
625  if (dv->isCollapsed()) {
626  // Restrict available domains to the ones in common with the operand.
627  // If there are no common domains, we must pay the cross-domain
628  // penalty for this operand.
629  if (common) available = common;
630  } else if (common)
631  // Open DomainValue is compatible, save it for merging.
632  used.push_back(rx);
633  else
634  // Open DomainValue is not compatible with instruction. It is useless
635  // now.
636  kill(rx);
637  }
638  }
639 
640  // If the collapsed operands force a single domain, propagate the collapse.
641  if (isPowerOf2_32(available)) {
642  unsigned domain = countTrailingZeros(available);
643  TII->setExecutionDomain(mi, domain);
644  visitHardInstr(mi, domain);
645  return;
646  }
647 
648  // Kill off any remaining uses that don't match available, and build a list of
649  // incoming DomainValues that we want to merge.
651  for (SmallVectorImpl<int>::iterator i=used.begin(), e=used.end(); i!=e; ++i) {
652  int rx = *i;
653  assert(LiveRegs && "no space allocated for live registers");
654  const LiveReg &LR = LiveRegs[rx];
655  // This useless DomainValue could have been missed above.
656  if (!LR.Value->getCommonDomains(available)) {
657  kill(rx);
658  continue;
659  }
660  // Sorted insertion.
661  bool Inserted = false;
662  for (SmallVectorImpl<LiveReg>::iterator i = Regs.begin(), e = Regs.end();
663  i != e && !Inserted; ++i) {
664  if (LR.Def < i->Def) {
665  Inserted = true;
666  Regs.insert(i, LR);
667  }
668  }
669  if (!Inserted)
670  Regs.push_back(LR);
671  }
672 
673  // doms are now sorted in order of appearance. Try to merge them all, giving
674  // priority to the latest ones.
675  DomainValue *dv = nullptr;
676  while (!Regs.empty()) {
677  if (!dv) {
678  dv = Regs.pop_back_val().Value;
679  // Force the first dv to match the current instruction.
680  dv->AvailableDomains = dv->getCommonDomains(available);
681  assert(dv->AvailableDomains && "Domain should have been filtered");
682  continue;
683  }
684 
685  DomainValue *Latest = Regs.pop_back_val().Value;
686  // Skip already merged values.
687  if (Latest == dv || Latest->Next)
688  continue;
689  if (merge(dv, Latest))
690  continue;
691 
692  // If latest didn't merge, it is useless now. Kill all registers using it.
693  for (int i : used) {
694  assert(LiveRegs && "no space allocated for live registers");
695  if (LiveRegs[i].Value == Latest)
696  kill(i);
697  }
698  }
699 
700  // dv is the DomainValue we are going to use for this instruction.
701  if (!dv) {
702  dv = alloc();
703  dv->AvailableDomains = available;
704  }
705  dv->Instrs.push_back(mi);
706 
707  // Finally set all defs and non-collapsed uses to dv. We must iterate through
708  // all the operators, including imp-def ones.
710  ee = mi->operands_end();
711  ii != ee; ++ii) {
712  MachineOperand &mo = *ii;
713  if (!mo.isReg()) continue;
714  for (int rx : regIndices(mo.getReg())) {
715  if (!LiveRegs[rx].Value || (mo.isDef() && LiveRegs[rx].Value != dv)) {
716  kill(rx);
717  setLiveReg(rx, dv);
718  }
719  }
720  }
721 }
722 
723 bool ExeDepsFix::runOnMachineFunction(MachineFunction &mf) {
724  MF = &mf;
725  TII = MF->getSubtarget().getInstrInfo();
726  TRI = MF->getSubtarget().getRegisterInfo();
727  LiveRegs = nullptr;
728  assert(NumRegs == RC->getNumRegs() && "Bad regclass");
729 
730  DEBUG(dbgs() << "********** FIX EXECUTION DEPENDENCIES: "
731  << TRI->getRegClassName(RC) << " **********\n");
732 
733  // If no relevant registers are used in the function, we can skip it
734  // completely.
735  bool anyregs = false;
736  for (TargetRegisterClass::const_iterator I = RC->begin(), E = RC->end();
737  I != E; ++I)
738  if (MF->getRegInfo().isPhysRegUsed(*I)) {
739  anyregs = true;
740  break;
741  }
742  if (!anyregs) return false;
743 
744  // Initialize the AliasMap on the first use.
745  if (AliasMap.empty()) {
746  // Given a PhysReg, AliasMap[PhysReg] returns a list of indices into RC and
747  // therefore the LiveRegs array.
748  AliasMap.resize(TRI->getNumRegs());
749  for (unsigned i = 0, e = RC->getNumRegs(); i != e; ++i)
750  for (MCRegAliasIterator AI(RC->getRegister(i), TRI, true);
751  AI.isValid(); ++AI)
752  AliasMap[*AI].push_back(i);
753  }
754 
755  MachineBasicBlock *Entry = MF->begin();
759  MBBI = RPOT.begin(), MBBE = RPOT.end(); MBBI != MBBE; ++MBBI) {
760  MachineBasicBlock *MBB = *MBBI;
761  enterBasicBlock(MBB);
762  if (SeenUnknownBackEdge)
763  Loops.push_back(MBB);
764  for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;
765  ++I)
766  visitInstr(I);
767  processUndefReads(MBB);
768  leaveBasicBlock(MBB);
769  }
770 
771  // Visit all the loop blocks again in order to merge DomainValues from
772  // back-edges.
773  for (unsigned i = 0, e = Loops.size(); i != e; ++i) {
774  MachineBasicBlock *MBB = Loops[i];
775  enterBasicBlock(MBB);
776  for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;
777  ++I)
778  if (!I->isDebugValue())
779  processDefs(I, false);
780  processUndefReads(MBB);
781  leaveBasicBlock(MBB);
782  }
783 
784  // Clear the LiveOuts vectors and collapse any remaining DomainValues.
786  MBBI = RPOT.begin(), MBBE = RPOT.end(); MBBI != MBBE; ++MBBI) {
787  LiveOutMap::const_iterator FI = LiveOuts.find(*MBBI);
788  if (FI == LiveOuts.end() || !FI->second)
789  continue;
790  for (unsigned i = 0, e = NumRegs; i != e; ++i)
791  if (FI->second[i].Value)
792  release(FI->second[i].Value);
793  delete[] FI->second;
794  }
795  LiveOuts.clear();
796  UndefReads.clear();
797  Avail.clear();
798  Allocator.DestroyAll();
799 
800  return false;
801 }
802 
803 FunctionPass *
805  return new ExeDepsFix(RC);
806 }
bool isImplicit() const
void push_back(const T &Elt)
Definition: SmallVector.h:222
const MCPhysReg * const_iterator
mop_iterator operands_end()
Definition: MachineInstr.h:290
int getNumber() const
getNumber - MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a M...
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:191
std::vector< unsigned >::const_iterator livein_iterator
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:138
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:264
This file defines the MallocAllocator and BumpPtrAllocator interfaces.
livein_iterator livein_begin() const
Hexagon Hardware Loops
global merge
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
Definition: SmallVector.h:406
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Reg
All possible values of the reg field in the ModR/M byte.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
Definition: MachineInstr.h:271
const HexagonRegisterInfo & getRegisterInfo() const
getRegisterInfo - TargetInstrInfo is a superset of MRegister info.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
reverse_iterator rend()
reverse_iterator rbegin()
TargetInstrInfo - Interface to description of machine instruction set.
bool isDebugValue() const
Definition: MachineInstr.h:748
bundle_iterator< MachineInstr, instr_iterator > iterator
#define rc(i)
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:109
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
livein_iterator livein_end() const
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:273
A set of live virtual registers and physical register units.
MCRegAliasIterator enumerates all registers aliasing Reg.
Represent the analysis usage information of a pass.
bool isVariadic(QueryType Type=IgnoreBundle) const
Return true if this instruction can have a variable number of operands.
Definition: MachineInstr.h:383
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:294
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
std::vector< NodeType * >::reverse_iterator rpo_iterator
std::vector< MachineBasicBlock * >::const_iterator const_pred_iterator
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
FunctionPass * createExecutionDependencyFixPass(const TargetRegisterClass *RC)
createExecutionDependencyFixPass - This pass fixes execution time problems with dependent instruction...
A BumpPtrAllocator that allows only elements of a specific type to be allocated.
Definition: Allocator.h:349
bool contains(unsigned Reg) const
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
A range adaptor for a pair of iterators.
void setPreservesAll()
Set by analyses that do not transform their input at all.
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:481
Representation of each machine instruction.
Definition: MachineInstr.h:51
A set of live physical registers with functions to track liveness when walking backward/forward throu...
Definition: LivePhysRegs.h:43
#define I(x, y, z)
Definition: MD5.cpp:54
unsigned getReg() const
getReg - Returns the register number.
std::reverse_iterator< iterator > reverse_iterator
LLVM Value Representation.
Definition: Value.h:69
mop_iterator operands_begin()
Definition: MachineInstr.h:289
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:185
#define DEBUG(X)
Definition: Debug.h:92
bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
Definition: MathExtras.h:354