LLVM  3.7.0
RegisterCoalescer.cpp
Go to the documentation of this file.
1 //===- RegisterCoalescer.cpp - Generic Register Coalescing Interface -------==//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the generic RegisterCoalescer interface which
11 // is used as the common interface used by all clients and
12 // implementations of register coalescing.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "RegisterCoalescer.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/Statistic.h"
27 #include "llvm/CodeGen/Passes.h"
30 #include "llvm/IR/Value.h"
31 #include "llvm/Pass.h"
33 #include "llvm/Support/Debug.h"
35 #include "llvm/Support/Format.h"
41 #include <algorithm>
42 #include <cmath>
43 using namespace llvm;
44 
45 #define DEBUG_TYPE "regalloc"
46 
47 STATISTIC(numJoins , "Number of interval joins performed");
48 STATISTIC(numCrossRCs , "Number of cross class joins performed");
49 STATISTIC(numCommutes , "Number of instruction commuting performed");
50 STATISTIC(numExtends , "Number of copies extended");
51 STATISTIC(NumReMats , "Number of instructions re-materialized");
52 STATISTIC(NumInflated , "Number of register classes inflated");
53 STATISTIC(NumLaneConflicts, "Number of dead lane conflicts tested");
54 STATISTIC(NumLaneResolves, "Number of dead lane conflicts resolved");
55 
56 static cl::opt<bool>
57 EnableJoining("join-liveintervals",
58  cl::desc("Coalesce copies (default=true)"),
59  cl::init(true));
60 
61 static cl::opt<bool> UseTerminalRule("terminal-rule",
62  cl::desc("Apply the terminal rule"),
63  cl::init(false), cl::Hidden);
64 
65 /// Temporary flag to test critical edge unsplitting.
66 static cl::opt<bool>
67 EnableJoinSplits("join-splitedges",
68  cl::desc("Coalesce copies on split edges (default=subtarget)"), cl::Hidden);
69 
70 /// Temporary flag to test global copy optimization.
72 EnableGlobalCopies("join-globalcopies",
73  cl::desc("Coalesce copies that span blocks (default=subtarget)"),
75 
76 static cl::opt<bool>
77 VerifyCoalescing("verify-coalescing",
78  cl::desc("Verify machine instrs before and after register coalescing"),
79  cl::Hidden);
80 
81 namespace {
82  class RegisterCoalescer : public MachineFunctionPass,
83  private LiveRangeEdit::Delegate {
84  MachineFunction* MF;
86  const TargetMachine* TM;
87  const TargetRegisterInfo* TRI;
88  const TargetInstrInfo* TII;
89  LiveIntervals *LIS;
90  const MachineLoopInfo* Loops;
91  AliasAnalysis *AA;
92  RegisterClassInfo RegClassInfo;
93 
94  /// A LaneMask to remember on which subregister live ranges we need to call
95  /// shrinkToUses() later.
96  unsigned ShrinkMask;
97 
98  /// True if the main range of the currently coalesced intervals should be
99  /// checked for smaller live intervals.
100  bool ShrinkMainRange;
101 
102  /// \brief True if the coalescer should aggressively coalesce global copies
103  /// in favor of keeping local copies.
104  bool JoinGlobalCopies;
105 
106  /// \brief True if the coalescer should aggressively coalesce fall-thru
107  /// blocks exclusively containing copies.
108  bool JoinSplitEdges;
109 
110  /// Copy instructions yet to be coalesced.
112  SmallVector<MachineInstr*, 8> LocalWorkList;
113 
114  /// Set of instruction pointers that have been erased, and
115  /// that may be present in WorkList.
116  SmallPtrSet<MachineInstr*, 8> ErasedInstrs;
117 
118  /// Dead instructions that are about to be deleted.
120 
121  /// Virtual registers to be considered for register class inflation.
122  SmallVector<unsigned, 8> InflateRegs;
123 
124  /// Recursively eliminate dead defs in DeadDefs.
125  void eliminateDeadDefs();
126 
127  /// LiveRangeEdit callback for eliminateDeadDefs().
128  void LRE_WillEraseInstruction(MachineInstr *MI) override;
129 
130  /// Coalesce the LocalWorkList.
131  void coalesceLocals();
132 
133  /// Join compatible live intervals
134  void joinAllIntervals();
135 
136  /// Coalesce copies in the specified MBB, putting
137  /// copies that cannot yet be coalesced into WorkList.
138  void copyCoalesceInMBB(MachineBasicBlock *MBB);
139 
140  /// Tries to coalesce all copies in CurrList. Returns true if any progress
141  /// was made.
142  bool copyCoalesceWorkList(MutableArrayRef<MachineInstr*> CurrList);
143 
144  /// Attempt to join intervals corresponding to SrcReg/DstReg, which are the
145  /// src/dst of the copy instruction CopyMI. This returns true if the copy
146  /// was successfully coalesced away. If it is not currently possible to
147  /// coalesce this interval, but it may be possible if other things get
148  /// coalesced, then it returns true by reference in 'Again'.
149  bool joinCopy(MachineInstr *TheCopy, bool &Again);
150 
151  /// Attempt to join these two intervals. On failure, this
152  /// returns false. The output "SrcInt" will not have been modified, so we
153  /// can use this information below to update aliases.
154  bool joinIntervals(CoalescerPair &CP);
155 
156  /// Attempt joining two virtual registers. Return true on success.
157  bool joinVirtRegs(CoalescerPair &CP);
158 
159  /// Attempt joining with a reserved physreg.
160  bool joinReservedPhysReg(CoalescerPair &CP);
161 
162  /// Add the LiveRange @p ToMerge as a subregister liverange of @p LI.
163  /// Subranges in @p LI which only partially interfere with the desired
164  /// LaneMask are split as necessary. @p LaneMask are the lanes that
165  /// @p ToMerge will occupy in the coalescer register. @p LI has its subrange
166  /// lanemasks already adjusted to the coalesced register.
167  /// @returns false if live range conflicts couldn't get resolved.
168  bool mergeSubRangeInto(LiveInterval &LI, const LiveRange &ToMerge,
169  unsigned LaneMask, CoalescerPair &CP);
170 
171  /// Join the liveranges of two subregisters. Joins @p RRange into
172  /// @p LRange, @p RRange may be invalid afterwards.
173  /// @returns false if live range conflicts couldn't get resolved.
174  bool joinSubRegRanges(LiveRange &LRange, LiveRange &RRange,
175  unsigned LaneMask, const CoalescerPair &CP);
176 
177  /// We found a non-trivially-coalescable copy. If the source value number is
178  /// defined by a copy from the destination reg see if we can merge these two
179  /// destination reg valno# into a single value number, eliminating a copy.
180  /// This returns true if an interval was modified.
181  bool adjustCopiesBackFrom(const CoalescerPair &CP, MachineInstr *CopyMI);
182 
183  /// Return true if there are definitions of IntB
184  /// other than BValNo val# that can reach uses of AValno val# of IntA.
185  bool hasOtherReachingDefs(LiveInterval &IntA, LiveInterval &IntB,
186  VNInfo *AValNo, VNInfo *BValNo);
187 
188  /// We found a non-trivially-coalescable copy.
189  /// If the source value number is defined by a commutable instruction and
190  /// its other operand is coalesced to the copy dest register, see if we
191  /// can transform the copy into a noop by commuting the definition.
192  /// This returns true if an interval was modified.
193  bool removeCopyByCommutingDef(const CoalescerPair &CP,MachineInstr *CopyMI);
194 
195  /// If the source of a copy is defined by a
196  /// trivial computation, replace the copy by rematerialize the definition.
197  bool reMaterializeTrivialDef(const CoalescerPair &CP, MachineInstr *CopyMI,
198  bool &IsDefCopy);
199 
200  /// Return true if a copy involving a physreg should be joined.
201  bool canJoinPhys(const CoalescerPair &CP);
202 
203  /// Replace all defs and uses of SrcReg to DstReg and update the subregister
204  /// number if it is not zero. If DstReg is a physical register and the
205  /// existing subregister number of the def / use being updated is not zero,
206  /// make sure to set it to the correct physical subregister.
207  void updateRegDefsUses(unsigned SrcReg, unsigned DstReg, unsigned SubIdx);
208 
209  /// Handle copies of undef values.
210  /// Returns true if @p CopyMI was a copy of an undef value and eliminated.
211  bool eliminateUndefCopy(MachineInstr *CopyMI);
212 
213  /// Check whether or not we should apply the terminal rule on the
214  /// destination (Dst) of \p Copy.
215  /// When the terminal rule applies, Copy is not profitable to
216  /// coalesce.
217  /// Dst is terminal if it has exactly one affinity (Dst, Src) and
218  /// at least one interference (Dst, Dst2). If Dst is terminal, the
219  /// terminal rule consists in checking that at least one of
220  /// interfering node, say Dst2, has an affinity of equal or greater
221  /// weight with Src.
222  /// In that case, Dst2 and Dst will not be able to be both coalesced
223  /// with Src. Since Dst2 exposes more coalescing opportunities than
224  /// Dst, we can drop \p Copy.
225  bool applyTerminalRule(const MachineInstr &Copy) const;
226 
227  /// Check whether or not \p LI is composed by multiple connected
228  /// components and if that is the case, fix that.
229  void splitNewRanges(LiveInterval *LI) {
230  ConnectedVNInfoEqClasses ConEQ(*LIS);
231  unsigned NumComps = ConEQ.Classify(LI);
232  if (NumComps <= 1)
233  return;
234  SmallVector<LiveInterval*, 8> NewComps(1, LI);
235  for (unsigned i = 1; i != NumComps; ++i) {
236  unsigned VReg = MRI->createVirtualRegister(MRI->getRegClass(LI->reg));
237  NewComps.push_back(&LIS->createEmptyInterval(VReg));
238  }
239 
240  ConEQ.Distribute(&NewComps[0], *MRI);
241  }
242 
243  /// Wrapper method for \see LiveIntervals::shrinkToUses.
244  /// This method does the proper fixing of the live-ranges when the afore
245  /// mentioned method returns true.
246  void shrinkToUses(LiveInterval *LI,
248  if (LIS->shrinkToUses(LI, Dead))
249  // We may have created multiple connected components, split them.
250  splitNewRanges(LI);
251  }
252 
253  public:
254  static char ID; ///< Class identification, replacement for typeinfo
255  RegisterCoalescer() : MachineFunctionPass(ID) {
257  }
258 
259  void getAnalysisUsage(AnalysisUsage &AU) const override;
260 
261  void releaseMemory() override;
262 
263  /// This is the pass entry point.
264  bool runOnMachineFunction(MachineFunction&) override;
265 
266  /// Implement the dump method.
267  void print(raw_ostream &O, const Module* = nullptr) const override;
268  };
269 } // end anonymous namespace
270 
272 
273 INITIALIZE_PASS_BEGIN(RegisterCoalescer, "simple-register-coalescing",
274  "Simple Register Coalescing", false, false)
279 INITIALIZE_PASS_END(RegisterCoalescer, "simple-register-coalescing",
280  "Simple Register Coalescing", false, false)
281 
282 char RegisterCoalescer::ID = 0;
283 
285  unsigned &Src, unsigned &Dst,
286  unsigned &SrcSub, unsigned &DstSub) {
287  if (MI->isCopy()) {
288  Dst = MI->getOperand(0).getReg();
289  DstSub = MI->getOperand(0).getSubReg();
290  Src = MI->getOperand(1).getReg();
291  SrcSub = MI->getOperand(1).getSubReg();
292  } else if (MI->isSubregToReg()) {
293  Dst = MI->getOperand(0).getReg();
294  DstSub = tri.composeSubRegIndices(MI->getOperand(0).getSubReg(),
295  MI->getOperand(3).getImm());
296  Src = MI->getOperand(2).getReg();
297  SrcSub = MI->getOperand(2).getSubReg();
298  } else
299  return false;
300  return true;
301 }
302 
303 /// Return true if this block should be vacated by the coalescer to eliminate
304 /// branches. The important cases to handle in the coalescer are critical edges
305 /// split during phi elimination which contain only copies. Simple blocks that
306 /// contain non-branches should also be vacated, but this can be handled by an
307 /// earlier pass similar to early if-conversion.
308 static bool isSplitEdge(const MachineBasicBlock *MBB) {
309  if (MBB->pred_size() != 1 || MBB->succ_size() != 1)
310  return false;
311 
312  for (const auto &MI : *MBB) {
313  if (!MI.isCopyLike() && !MI.isUnconditionalBranch())
314  return false;
315  }
316  return true;
317 }
318 
320  SrcReg = DstReg = 0;
321  SrcIdx = DstIdx = 0;
322  NewRC = nullptr;
323  Flipped = CrossClass = false;
324 
325  unsigned Src, Dst, SrcSub, DstSub;
326  if (!isMoveInstr(TRI, MI, Src, Dst, SrcSub, DstSub))
327  return false;
328  Partial = SrcSub || DstSub;
329 
330  // If one register is a physreg, it must be Dst.
333  return false;
334  std::swap(Src, Dst);
335  std::swap(SrcSub, DstSub);
336  Flipped = true;
337  }
338 
339  const MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo();
340 
342  // Eliminate DstSub on a physreg.
343  if (DstSub) {
344  Dst = TRI.getSubReg(Dst, DstSub);
345  if (!Dst) return false;
346  DstSub = 0;
347  }
348 
349  // Eliminate SrcSub by picking a corresponding Dst superregister.
350  if (SrcSub) {
351  Dst = TRI.getMatchingSuperReg(Dst, SrcSub, MRI.getRegClass(Src));
352  if (!Dst) return false;
353  } else if (!MRI.getRegClass(Src)->contains(Dst)) {
354  return false;
355  }
356  } else {
357  // Both registers are virtual.
358  const TargetRegisterClass *SrcRC = MRI.getRegClass(Src);
359  const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
360 
361  // Both registers have subreg indices.
362  if (SrcSub && DstSub) {
363  // Copies between different sub-registers are never coalescable.
364  if (Src == Dst && SrcSub != DstSub)
365  return false;
366 
367  NewRC = TRI.getCommonSuperRegClass(SrcRC, SrcSub, DstRC, DstSub,
368  SrcIdx, DstIdx);
369  if (!NewRC)
370  return false;
371  } else if (DstSub) {
372  // SrcReg will be merged with a sub-register of DstReg.
373  SrcIdx = DstSub;
374  NewRC = TRI.getMatchingSuperRegClass(DstRC, SrcRC, DstSub);
375  } else if (SrcSub) {
376  // DstReg will be merged with a sub-register of SrcReg.
377  DstIdx = SrcSub;
378  NewRC = TRI.getMatchingSuperRegClass(SrcRC, DstRC, SrcSub);
379  } else {
380  // This is a straight copy without sub-registers.
381  NewRC = TRI.getCommonSubClass(DstRC, SrcRC);
382  }
383 
384  // The combined constraint may be impossible to satisfy.
385  if (!NewRC)
386  return false;
387 
388  // Prefer SrcReg to be a sub-register of DstReg.
389  // FIXME: Coalescer should support subregs symmetrically.
390  if (DstIdx && !SrcIdx) {
391  std::swap(Src, Dst);
392  std::swap(SrcIdx, DstIdx);
393  Flipped = !Flipped;
394  }
395 
396  CrossClass = NewRC != DstRC || NewRC != SrcRC;
397  }
398  // Check our invariants
399  assert(TargetRegisterInfo::isVirtualRegister(Src) && "Src must be virtual");
400  assert(!(TargetRegisterInfo::isPhysicalRegister(Dst) && DstSub) &&
401  "Cannot have a physical SubIdx");
402  SrcReg = Src;
403  DstReg = Dst;
404  return true;
405 }
406 
409  return false;
410  std::swap(SrcReg, DstReg);
411  std::swap(SrcIdx, DstIdx);
412  Flipped = !Flipped;
413  return true;
414 }
415 
417  if (!MI)
418  return false;
419  unsigned Src, Dst, SrcSub, DstSub;
420  if (!isMoveInstr(TRI, MI, Src, Dst, SrcSub, DstSub))
421  return false;
422 
423  // Find the virtual register that is SrcReg.
424  if (Dst == SrcReg) {
425  std::swap(Src, Dst);
426  std::swap(SrcSub, DstSub);
427  } else if (Src != SrcReg) {
428  return false;
429  }
430 
431  // Now check that Dst matches DstReg.
434  return false;
435  assert(!DstIdx && !SrcIdx && "Inconsistent CoalescerPair state.");
436  // DstSub could be set for a physreg from INSERT_SUBREG.
437  if (DstSub)
438  Dst = TRI.getSubReg(Dst, DstSub);
439  // Full copy of Src.
440  if (!SrcSub)
441  return DstReg == Dst;
442  // This is a partial register copy. Check that the parts match.
443  return TRI.getSubReg(DstReg, SrcSub) == Dst;
444  } else {
445  // DstReg is virtual.
446  if (DstReg != Dst)
447  return false;
448  // Registers match, do the subregisters line up?
449  return TRI.composeSubRegIndices(SrcIdx, SrcSub) ==
450  TRI.composeSubRegIndices(DstIdx, DstSub);
451  }
452 }
453 
454 void RegisterCoalescer::getAnalysisUsage(AnalysisUsage &AU) const {
455  AU.setPreservesCFG();
464 }
465 
466 void RegisterCoalescer::eliminateDeadDefs() {
467  SmallVector<unsigned, 8> NewRegs;
468  LiveRangeEdit(nullptr, NewRegs, *MF, *LIS,
469  nullptr, this).eliminateDeadDefs(DeadDefs);
470 }
471 
472 void RegisterCoalescer::LRE_WillEraseInstruction(MachineInstr *MI) {
473  // MI may be in WorkList. Make sure we don't visit it.
474  ErasedInstrs.insert(MI);
475 }
476 
477 bool RegisterCoalescer::adjustCopiesBackFrom(const CoalescerPair &CP,
478  MachineInstr *CopyMI) {
479  assert(!CP.isPartial() && "This doesn't work for partial copies.");
480  assert(!CP.isPhys() && "This doesn't work for physreg copies.");
481 
482  LiveInterval &IntA =
483  LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg());
484  LiveInterval &IntB =
485  LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg());
486  SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI).getRegSlot();
487 
488  // We have a non-trivially-coalescable copy with IntA being the source and
489  // IntB being the dest, thus this defines a value number in IntB. If the
490  // source value number (in IntA) is defined by a copy from B, see if we can
491  // merge these two pieces of B into a single value number, eliminating a copy.
492  // For example:
493  //
494  // A3 = B0
495  // ...
496  // B1 = A3 <- this copy
497  //
498  // In this case, B0 can be extended to where the B1 copy lives, allowing the
499  // B1 value number to be replaced with B0 (which simplifies the B
500  // liveinterval).
501 
502  // BValNo is a value number in B that is defined by a copy from A. 'B1' in
503  // the example above.
505  if (BS == IntB.end()) return false;
506  VNInfo *BValNo = BS->valno;
507 
508  // Get the location that B is defined at. Two options: either this value has
509  // an unknown definition point or it is defined at CopyIdx. If unknown, we
510  // can't process it.
511  if (BValNo->def != CopyIdx) return false;
512 
513  // AValNo is the value number in A that defines the copy, A3 in the example.
514  SlotIndex CopyUseIdx = CopyIdx.getRegSlot(true);
515  LiveInterval::iterator AS = IntA.FindSegmentContaining(CopyUseIdx);
516  // The live segment might not exist after fun with physreg coalescing.
517  if (AS == IntA.end()) return false;
518  VNInfo *AValNo = AS->valno;
519 
520  // If AValNo is defined as a copy from IntB, we can potentially process this.
521  // Get the instruction that defines this value number.
522  MachineInstr *ACopyMI = LIS->getInstructionFromIndex(AValNo->def);
523  // Don't allow any partial copies, even if isCoalescable() allows them.
524  if (!CP.isCoalescable(ACopyMI) || !ACopyMI->isFullCopy())
525  return false;
526 
527  // Get the Segment in IntB that this value number starts with.
529  IntB.FindSegmentContaining(AValNo->def.getPrevSlot());
530  if (ValS == IntB.end())
531  return false;
532 
533  // Make sure that the end of the live segment is inside the same block as
534  // CopyMI.
535  MachineInstr *ValSEndInst =
536  LIS->getInstructionFromIndex(ValS->end.getPrevSlot());
537  if (!ValSEndInst || ValSEndInst->getParent() != CopyMI->getParent())
538  return false;
539 
540  // Okay, we now know that ValS ends in the same block that the CopyMI
541  // live-range starts. If there are no intervening live segments between them
542  // in IntB, we can merge them.
543  if (ValS+1 != BS) return false;
544 
545  DEBUG(dbgs() << "Extending: " << PrintReg(IntB.reg, TRI));
546 
547  SlotIndex FillerStart = ValS->end, FillerEnd = BS->start;
548  // We are about to delete CopyMI, so need to remove it as the 'instruction
549  // that defines this value #'. Update the valnum with the new defining
550  // instruction #.
551  BValNo->def = FillerStart;
552 
553  // Okay, we can merge them. We need to insert a new liverange:
554  // [ValS.end, BS.begin) of either value number, then we merge the
555  // two value numbers.
556  IntB.addSegment(LiveInterval::Segment(FillerStart, FillerEnd, BValNo));
557 
558  // Okay, merge "B1" into the same value number as "B0".
559  if (BValNo != ValS->valno)
560  IntB.MergeValueNumberInto(BValNo, ValS->valno);
561 
562  // Do the same for the subregister segments.
563  for (LiveInterval::SubRange &S : IntB.subranges()) {
564  VNInfo *SubBValNo = S.getVNInfoAt(CopyIdx);
565  S.addSegment(LiveInterval::Segment(FillerStart, FillerEnd, SubBValNo));
566  VNInfo *SubValSNo = S.getVNInfoAt(AValNo->def.getPrevSlot());
567  if (SubBValNo != SubValSNo)
568  S.MergeValueNumberInto(SubBValNo, SubValSNo);
569  }
570 
571  DEBUG(dbgs() << " result = " << IntB << '\n');
572 
573  // If the source instruction was killing the source register before the
574  // merge, unset the isKill marker given the live range has been extended.
575  int UIdx = ValSEndInst->findRegisterUseOperandIdx(IntB.reg, true);
576  if (UIdx != -1) {
577  ValSEndInst->getOperand(UIdx).setIsKill(false);
578  }
579 
580  // Rewrite the copy. If the copy instruction was killing the destination
581  // register before the merge, find the last use and trim the live range. That
582  // will also add the isKill marker.
583  CopyMI->substituteRegister(IntA.reg, IntB.reg, 0, *TRI);
584  if (AS->end == CopyIdx)
585  shrinkToUses(&IntA);
586 
587  ++numExtends;
588  return true;
589 }
590 
591 bool RegisterCoalescer::hasOtherReachingDefs(LiveInterval &IntA,
592  LiveInterval &IntB,
593  VNInfo *AValNo,
594  VNInfo *BValNo) {
595  // If AValNo has PHI kills, conservatively assume that IntB defs can reach
596  // the PHI values.
597  if (LIS->hasPHIKill(IntA, AValNo))
598  return true;
599 
600  for (LiveRange::Segment &ASeg : IntA.segments) {
601  if (ASeg.valno != AValNo) continue;
603  std::upper_bound(IntB.begin(), IntB.end(), ASeg.start);
604  if (BI != IntB.begin())
605  --BI;
606  for (; BI != IntB.end() && ASeg.end >= BI->start; ++BI) {
607  if (BI->valno == BValNo)
608  continue;
609  if (BI->start <= ASeg.start && BI->end > ASeg.start)
610  return true;
611  if (BI->start > ASeg.start && BI->start < ASeg.end)
612  return true;
613  }
614  }
615  return false;
616 }
617 
618 /// Copy segements with value number @p SrcValNo from liverange @p Src to live
619 /// range @Dst and use value number @p DstValNo there.
620 static void addSegmentsWithValNo(LiveRange &Dst, VNInfo *DstValNo,
621  const LiveRange &Src, const VNInfo *SrcValNo)
622 {
623  for (const LiveRange::Segment &S : Src.segments) {
624  if (S.valno != SrcValNo)
625  continue;
626  Dst.addSegment(LiveRange::Segment(S.start, S.end, DstValNo));
627  }
628 }
629 
630 bool RegisterCoalescer::removeCopyByCommutingDef(const CoalescerPair &CP,
631  MachineInstr *CopyMI) {
632  assert(!CP.isPhys());
633 
634  LiveInterval &IntA =
635  LIS->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg());
636  LiveInterval &IntB =
637  LIS->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg());
638 
639  // We found a non-trivially-coalescable copy with IntA being the source and
640  // IntB being the dest, thus this defines a value number in IntB. If the
641  // source value number (in IntA) is defined by a commutable instruction and
642  // its other operand is coalesced to the copy dest register, see if we can
643  // transform the copy into a noop by commuting the definition. For example,
644  //
645  // A3 = op A2 B0<kill>
646  // ...
647  // B1 = A3 <- this copy
648  // ...
649  // = op A3 <- more uses
650  //
651  // ==>
652  //
653  // B2 = op B0 A2<kill>
654  // ...
655  // B1 = B2 <- now an identity copy
656  // ...
657  // = op B2 <- more uses
658 
659  // BValNo is a value number in B that is defined by a copy from A. 'B1' in
660  // the example above.
661  SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI).getRegSlot();
662  VNInfo *BValNo = IntB.getVNInfoAt(CopyIdx);
663  assert(BValNo != nullptr && BValNo->def == CopyIdx);
664 
665  // AValNo is the value number in A that defines the copy, A3 in the example.
666  VNInfo *AValNo = IntA.getVNInfoAt(CopyIdx.getRegSlot(true));
667  assert(AValNo && !AValNo->isUnused() && "COPY source not live");
668  if (AValNo->isPHIDef())
669  return false;
670  MachineInstr *DefMI = LIS->getInstructionFromIndex(AValNo->def);
671  if (!DefMI)
672  return false;
673  if (!DefMI->isCommutable())
674  return false;
675  // If DefMI is a two-address instruction then commuting it will change the
676  // destination register.
677  int DefIdx = DefMI->findRegisterDefOperandIdx(IntA.reg);
678  assert(DefIdx != -1);
679  unsigned UseOpIdx;
680  if (!DefMI->isRegTiedToUseOperand(DefIdx, &UseOpIdx))
681  return false;
682  unsigned Op1, Op2, NewDstIdx;
683  if (!TII->findCommutedOpIndices(DefMI, Op1, Op2))
684  return false;
685  if (Op1 == UseOpIdx)
686  NewDstIdx = Op2;
687  else if (Op2 == UseOpIdx)
688  NewDstIdx = Op1;
689  else
690  return false;
691 
692  MachineOperand &NewDstMO = DefMI->getOperand(NewDstIdx);
693  unsigned NewReg = NewDstMO.getReg();
694  if (NewReg != IntB.reg || !IntB.Query(AValNo->def).isKill())
695  return false;
696 
697  // Make sure there are no other definitions of IntB that would reach the
698  // uses which the new definition can reach.
699  if (hasOtherReachingDefs(IntA, IntB, AValNo, BValNo))
700  return false;
701 
702  // If some of the uses of IntA.reg is already coalesced away, return false.
703  // It's not possible to determine whether it's safe to perform the coalescing.
704  for (MachineOperand &MO : MRI->use_nodbg_operands(IntA.reg)) {
705  MachineInstr *UseMI = MO.getParent();
706  unsigned OpNo = &MO - &UseMI->getOperand(0);
707  SlotIndex UseIdx = LIS->getInstructionIndex(UseMI);
708  LiveInterval::iterator US = IntA.FindSegmentContaining(UseIdx);
709  if (US == IntA.end() || US->valno != AValNo)
710  continue;
711  // If this use is tied to a def, we can't rewrite the register.
712  if (UseMI->isRegTiedToDefOperand(OpNo))
713  return false;
714  }
715 
716  DEBUG(dbgs() << "\tremoveCopyByCommutingDef: " << AValNo->def << '\t'
717  << *DefMI);
718 
719  // At this point we have decided that it is legal to do this
720  // transformation. Start by commuting the instruction.
721  MachineBasicBlock *MBB = DefMI->getParent();
722  MachineInstr *NewMI = TII->commuteInstruction(DefMI);
723  if (!NewMI)
724  return false;
727  !MRI->constrainRegClass(IntB.reg, MRI->getRegClass(IntA.reg)))
728  return false;
729  if (NewMI != DefMI) {
730  LIS->ReplaceMachineInstrInMaps(DefMI, NewMI);
731  MachineBasicBlock::iterator Pos = DefMI;
732  MBB->insert(Pos, NewMI);
733  MBB->erase(DefMI);
734  }
735 
736  // If ALR and BLR overlaps and end of BLR extends beyond end of ALR, e.g.
737  // A = or A, B
738  // ...
739  // B = A
740  // ...
741  // C = A<kill>
742  // ...
743  // = B
744 
745  // Update uses of IntA of the specific Val# with IntB.
746  for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(IntA.reg),
747  UE = MRI->use_end();
748  UI != UE; /* ++UI is below because of possible MI removal */) {
749  MachineOperand &UseMO = *UI;
750  ++UI;
751  if (UseMO.isUndef())
752  continue;
753  MachineInstr *UseMI = UseMO.getParent();
754  if (UseMI->isDebugValue()) {
755  // FIXME These don't have an instruction index. Not clear we have enough
756  // info to decide whether to do this replacement or not. For now do it.
757  UseMO.setReg(NewReg);
758  continue;
759  }
760  SlotIndex UseIdx = LIS->getInstructionIndex(UseMI).getRegSlot(true);
761  LiveInterval::iterator US = IntA.FindSegmentContaining(UseIdx);
762  assert(US != IntA.end() && "Use must be live");
763  if (US->valno != AValNo)
764  continue;
765  // Kill flags are no longer accurate. They are recomputed after RA.
766  UseMO.setIsKill(false);
768  UseMO.substPhysReg(NewReg, *TRI);
769  else
770  UseMO.setReg(NewReg);
771  if (UseMI == CopyMI)
772  continue;
773  if (!UseMI->isCopy())
774  continue;
775  if (UseMI->getOperand(0).getReg() != IntB.reg ||
776  UseMI->getOperand(0).getSubReg())
777  continue;
778 
779  // This copy will become a noop. If it's defining a new val#, merge it into
780  // BValNo.
781  SlotIndex DefIdx = UseIdx.getRegSlot();
782  VNInfo *DVNI = IntB.getVNInfoAt(DefIdx);
783  if (!DVNI)
784  continue;
785  DEBUG(dbgs() << "\t\tnoop: " << DefIdx << '\t' << *UseMI);
786  assert(DVNI->def == DefIdx);
787  BValNo = IntB.MergeValueNumberInto(DVNI, BValNo);
788  for (LiveInterval::SubRange &S : IntB.subranges()) {
789  VNInfo *SubDVNI = S.getVNInfoAt(DefIdx);
790  if (!SubDVNI)
791  continue;
792  VNInfo *SubBValNo = S.getVNInfoAt(CopyIdx);
793  assert(SubBValNo->def == CopyIdx);
794  S.MergeValueNumberInto(SubDVNI, SubBValNo);
795  }
796 
797  ErasedInstrs.insert(UseMI);
798  LIS->RemoveMachineInstrFromMaps(UseMI);
799  UseMI->eraseFromParent();
800  }
801 
802  // Extend BValNo by merging in IntA live segments of AValNo. Val# definition
803  // is updated.
804  BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator();
805  if (IntB.hasSubRanges()) {
806  if (!IntA.hasSubRanges()) {
807  unsigned Mask = MRI->getMaxLaneMaskForVReg(IntA.reg);
808  IntA.createSubRangeFrom(Allocator, Mask, IntA);
809  }
810  SlotIndex AIdx = CopyIdx.getRegSlot(true);
811  for (LiveInterval::SubRange &SA : IntA.subranges()) {
812  VNInfo *ASubValNo = SA.getVNInfoAt(AIdx);
813  assert(ASubValNo != nullptr);
814 
815  unsigned AMask = SA.LaneMask;
816  for (LiveInterval::SubRange &SB : IntB.subranges()) {
817  unsigned BMask = SB.LaneMask;
818  unsigned Common = BMask & AMask;
819  if (Common == 0)
820  continue;
821 
822  DEBUG(
823  dbgs() << format("\t\tCopy+Merge %04X into %04X\n", BMask, Common));
824  unsigned BRest = BMask & ~AMask;
825  LiveInterval::SubRange *CommonRange;
826  if (BRest != 0) {
827  SB.LaneMask = BRest;
828  DEBUG(dbgs() << format("\t\tReduce Lane to %04X\n", BRest));
829  // Duplicate SubRange for newly merged common stuff.
830  CommonRange = IntB.createSubRangeFrom(Allocator, Common, SB);
831  } else {
832  // We van reuse the L SubRange.
833  SB.LaneMask = Common;
834  CommonRange = &SB;
835  }
836  LiveRange RangeCopy(SB, Allocator);
837 
838  VNInfo *BSubValNo = CommonRange->getVNInfoAt(CopyIdx);
839  assert(BSubValNo->def == CopyIdx);
840  BSubValNo->def = ASubValNo->def;
841  addSegmentsWithValNo(*CommonRange, BSubValNo, SA, ASubValNo);
842  AMask &= ~BMask;
843  }
844  if (AMask != 0) {
845  DEBUG(dbgs() << format("\t\tNew Lane %04X\n", AMask));
846  LiveRange *NewRange = IntB.createSubRange(Allocator, AMask);
847  VNInfo *BSubValNo = NewRange->getNextValue(CopyIdx, Allocator);
848  addSegmentsWithValNo(*NewRange, BSubValNo, SA, ASubValNo);
849  }
850  }
851  }
852 
853  BValNo->def = AValNo->def;
854  addSegmentsWithValNo(IntB, BValNo, IntA, AValNo);
855  DEBUG(dbgs() << "\t\textended: " << IntB << '\n');
856 
857  LIS->removeVRegDefAt(IntA, AValNo->def);
858 
859  DEBUG(dbgs() << "\t\ttrimmed: " << IntA << '\n');
860  ++numCommutes;
861  return true;
862 }
863 
864 /// Returns true if @p MI defines the full vreg @p Reg, as opposed to just
865 /// defining a subregister.
866 static bool definesFullReg(const MachineInstr &MI, unsigned Reg) {
868  "This code cannot handle physreg aliasing");
869  for (const MachineOperand &Op : MI.operands()) {
870  if (!Op.isReg() || !Op.isDef() || Op.getReg() != Reg)
871  continue;
872  // Return true if we define the full register or don't care about the value
873  // inside other subregisters.
874  if (Op.getSubReg() == 0 || Op.isUndef())
875  return true;
876  }
877  return false;
878 }
879 
880 bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP,
881  MachineInstr *CopyMI,
882  bool &IsDefCopy) {
883  IsDefCopy = false;
884  unsigned SrcReg = CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg();
885  unsigned SrcIdx = CP.isFlipped() ? CP.getDstIdx() : CP.getSrcIdx();
886  unsigned DstReg = CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg();
887  unsigned DstIdx = CP.isFlipped() ? CP.getSrcIdx() : CP.getDstIdx();
889  return false;
890 
891  LiveInterval &SrcInt = LIS->getInterval(SrcReg);
892  SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI);
893  VNInfo *ValNo = SrcInt.Query(CopyIdx).valueIn();
894  assert(ValNo && "CopyMI input register not live");
895  if (ValNo->isPHIDef() || ValNo->isUnused())
896  return false;
897  MachineInstr *DefMI = LIS->getInstructionFromIndex(ValNo->def);
898  if (!DefMI)
899  return false;
900  if (DefMI->isCopyLike()) {
901  IsDefCopy = true;
902  return false;
903  }
904  if (!TII->isAsCheapAsAMove(DefMI))
905  return false;
906  if (!TII->isTriviallyReMaterializable(DefMI, AA))
907  return false;
908  if (!definesFullReg(*DefMI, SrcReg))
909  return false;
910  bool SawStore = false;
911  if (!DefMI->isSafeToMove(AA, SawStore))
912  return false;
913  const MCInstrDesc &MCID = DefMI->getDesc();
914  if (MCID.getNumDefs() != 1)
915  return false;
916  // Only support subregister destinations when the def is read-undef.
917  MachineOperand &DstOperand = CopyMI->getOperand(0);
918  unsigned CopyDstReg = DstOperand.getReg();
919  if (DstOperand.getSubReg() && !DstOperand.isUndef())
920  return false;
921 
922  // If both SrcIdx and DstIdx are set, correct rematerialization would widen
923  // the register substantially (beyond both source and dest size). This is bad
924  // for performance since it can cascade through a function, introducing many
925  // extra spills and fills (e.g. ARM can easily end up copying QQQQPR registers
926  // around after a few subreg copies).
927  if (SrcIdx && DstIdx)
928  return false;
929 
930  const TargetRegisterClass *DefRC = TII->getRegClass(MCID, 0, TRI, *MF);
931  if (!DefMI->isImplicitDef()) {
933  unsigned NewDstReg = DstReg;
934 
935  unsigned NewDstIdx = TRI->composeSubRegIndices(CP.getSrcIdx(),
936  DefMI->getOperand(0).getSubReg());
937  if (NewDstIdx)
938  NewDstReg = TRI->getSubReg(DstReg, NewDstIdx);
939 
940  // Finally, make sure that the physical subregister that will be
941  // constructed later is permitted for the instruction.
942  if (!DefRC->contains(NewDstReg))
943  return false;
944  } else {
945  // Theoretically, some stack frame reference could exist. Just make sure
946  // it hasn't actually happened.
947  assert(TargetRegisterInfo::isVirtualRegister(DstReg) &&
948  "Only expect to deal with virtual or physical registers");
949  }
950  }
951 
952  MachineBasicBlock *MBB = CopyMI->getParent();
954  std::next(MachineBasicBlock::iterator(CopyMI));
955  TII->reMaterialize(*MBB, MII, DstReg, SrcIdx, DefMI, *TRI);
956  MachineInstr *NewMI = std::prev(MII);
957 
958  // In a situation like the following:
959  // %vreg0:subreg = instr ; DefMI, subreg = DstIdx
960  // %vreg1 = copy %vreg0:subreg ; CopyMI, SrcIdx = 0
961  // instead of widening %vreg1 to the register class of %vreg0 simply do:
962  // %vreg1 = instr
963  const TargetRegisterClass *NewRC = CP.getNewRC();
964  if (DstIdx != 0) {
965  MachineOperand &DefMO = NewMI->getOperand(0);
966  if (DefMO.getSubReg() == DstIdx) {
967  assert(SrcIdx == 0 && CP.isFlipped()
968  && "Shouldn't have SrcIdx+DstIdx at this point");
969  const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg);
970  const TargetRegisterClass *CommonRC =
971  TRI->getCommonSubClass(DefRC, DstRC);
972  if (CommonRC != nullptr) {
973  NewRC = CommonRC;
974  DstIdx = 0;
975  DefMO.setSubReg(0);
976  }
977  }
978  }
979 
980  LIS->ReplaceMachineInstrInMaps(CopyMI, NewMI);
981  CopyMI->eraseFromParent();
982  ErasedInstrs.insert(CopyMI);
983 
984  // NewMI may have dead implicit defs (E.g. EFLAGS for MOV<bits>r0 on X86).
985  // We need to remember these so we can add intervals once we insert
986  // NewMI into SlotIndexes.
987  SmallVector<unsigned, 4> NewMIImplDefs;
988  for (unsigned i = NewMI->getDesc().getNumOperands(),
989  e = NewMI->getNumOperands(); i != e; ++i) {
990  MachineOperand &MO = NewMI->getOperand(i);
991  if (MO.isReg() && MO.isDef()) {
992  assert(MO.isImplicit() && MO.isDead() &&
994  NewMIImplDefs.push_back(MO.getReg());
995  }
996  }
997 
999  unsigned NewIdx = NewMI->getOperand(0).getSubReg();
1000 
1001  if (DefRC != nullptr) {
1002  if (NewIdx)
1003  NewRC = TRI->getMatchingSuperRegClass(NewRC, DefRC, NewIdx);
1004  else
1005  NewRC = TRI->getCommonSubClass(NewRC, DefRC);
1006  assert(NewRC && "subreg chosen for remat incompatible with instruction");
1007  }
1008  MRI->setRegClass(DstReg, NewRC);
1009 
1010  updateRegDefsUses(DstReg, DstReg, DstIdx);
1011  NewMI->getOperand(0).setSubReg(NewIdx);
1012  } else if (NewMI->getOperand(0).getReg() != CopyDstReg) {
1013  // The New instruction may be defining a sub-register of what's actually
1014  // been asked for. If so it must implicitly define the whole thing.
1015  assert(TargetRegisterInfo::isPhysicalRegister(DstReg) &&
1016  "Only expect virtual or physical registers in remat");
1017  NewMI->getOperand(0).setIsDead(true);
1018  NewMI->addOperand(MachineOperand::CreateReg(CopyDstReg,
1019  true /*IsDef*/,
1020  true /*IsImp*/,
1021  false /*IsKill*/));
1022  // Record small dead def live-ranges for all the subregisters
1023  // of the destination register.
1024  // Otherwise, variables that live through may miss some
1025  // interferences, thus creating invalid allocation.
1026  // E.g., i386 code:
1027  // vreg1 = somedef ; vreg1 GR8
1028  // vreg2 = remat ; vreg2 GR32
1029  // CL = COPY vreg2.sub_8bit
1030  // = somedef vreg1 ; vreg1 GR8
1031  // =>
1032  // vreg1 = somedef ; vreg1 GR8
1033  // ECX<def, dead> = remat ; CL<imp-def>
1034  // = somedef vreg1 ; vreg1 GR8
1035  // vreg1 will see the inteferences with CL but not with CH since
1036  // no live-ranges would have been created for ECX.
1037  // Fix that!
1038  SlotIndex NewMIIdx = LIS->getInstructionIndex(NewMI);
1039  for (MCRegUnitIterator Units(NewMI->getOperand(0).getReg(), TRI);
1040  Units.isValid(); ++Units)
1041  if (LiveRange *LR = LIS->getCachedRegUnit(*Units))
1042  LR->createDeadDef(NewMIIdx.getRegSlot(), LIS->getVNInfoAllocator());
1043  }
1044 
1045  if (NewMI->getOperand(0).getSubReg())
1046  NewMI->getOperand(0).setIsUndef();
1047 
1048  // CopyMI may have implicit operands, transfer them over to the newly
1049  // rematerialized instruction. And update implicit def interval valnos.
1050  for (unsigned i = CopyMI->getDesc().getNumOperands(),
1051  e = CopyMI->getNumOperands(); i != e; ++i) {
1052  MachineOperand &MO = CopyMI->getOperand(i);
1053  if (MO.isReg()) {
1054  assert(MO.isImplicit() && "No explicit operands after implict operands.");
1055  // Discard VReg implicit defs.
1057  NewMI->addOperand(MO);
1058  }
1059  }
1060  }
1061 
1062  SlotIndex NewMIIdx = LIS->getInstructionIndex(NewMI);
1063  for (unsigned i = 0, e = NewMIImplDefs.size(); i != e; ++i) {
1064  unsigned Reg = NewMIImplDefs[i];
1065  for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units)
1066  if (LiveRange *LR = LIS->getCachedRegUnit(*Units))
1067  LR->createDeadDef(NewMIIdx.getRegSlot(), LIS->getVNInfoAllocator());
1068  }
1069 
1070  DEBUG(dbgs() << "Remat: " << *NewMI);
1071  ++NumReMats;
1072 
1073  // The source interval can become smaller because we removed a use.
1074  shrinkToUses(&SrcInt, &DeadDefs);
1075  if (!DeadDefs.empty()) {
1076  // If the virtual SrcReg is completely eliminated, update all DBG_VALUEs
1077  // to describe DstReg instead.
1078  for (MachineOperand &UseMO : MRI->use_operands(SrcReg)) {
1079  MachineInstr *UseMI = UseMO.getParent();
1080  if (UseMI->isDebugValue()) {
1081  UseMO.setReg(DstReg);
1082  DEBUG(dbgs() << "\t\tupdated: " << *UseMI);
1083  }
1084  }
1085  eliminateDeadDefs();
1086  }
1087 
1088  return true;
1089 }
1090 
1091 bool RegisterCoalescer::eliminateUndefCopy(MachineInstr *CopyMI) {
1092  // ProcessImpicitDefs may leave some copies of <undef> values, it only removes
1093  // local variables. When we have a copy like:
1094  //
1095  // %vreg1 = COPY %vreg2<undef>
1096  //
1097  // We delete the copy and remove the corresponding value number from %vreg1.
1098  // Any uses of that value number are marked as <undef>.
1099 
1100  // Note that we do not query CoalescerPair here but redo isMoveInstr as the
1101  // CoalescerPair may have a new register class with adjusted subreg indices
1102  // at this point.
1103  unsigned SrcReg, DstReg, SrcSubIdx, DstSubIdx;
1104  isMoveInstr(*TRI, CopyMI, SrcReg, DstReg, SrcSubIdx, DstSubIdx);
1105 
1106  SlotIndex Idx = LIS->getInstructionIndex(CopyMI);
1107  const LiveInterval &SrcLI = LIS->getInterval(SrcReg);
1108  // CopyMI is undef iff SrcReg is not live before the instruction.
1109  if (SrcSubIdx != 0 && SrcLI.hasSubRanges()) {
1110  unsigned SrcMask = TRI->getSubRegIndexLaneMask(SrcSubIdx);
1111  for (const LiveInterval::SubRange &SR : SrcLI.subranges()) {
1112  if ((SR.LaneMask & SrcMask) == 0)
1113  continue;
1114  if (SR.liveAt(Idx))
1115  return false;
1116  }
1117  } else if (SrcLI.liveAt(Idx))
1118  return false;
1119 
1120  DEBUG(dbgs() << "\tEliminating copy of <undef> value\n");
1121 
1122  // Remove any DstReg segments starting at the instruction.
1123  LiveInterval &DstLI = LIS->getInterval(DstReg);
1124  SlotIndex RegIndex = Idx.getRegSlot();
1125  // Remove value or merge with previous one in case of a subregister def.
1126  if (VNInfo *PrevVNI = DstLI.getVNInfoAt(Idx)) {
1127  VNInfo *VNI = DstLI.getVNInfoAt(RegIndex);
1128  DstLI.MergeValueNumberInto(VNI, PrevVNI);
1129 
1130  // The affected subregister segments can be removed.
1131  unsigned DstMask = TRI->getSubRegIndexLaneMask(DstSubIdx);
1132  for (LiveInterval::SubRange &SR : DstLI.subranges()) {
1133  if ((SR.LaneMask & DstMask) == 0)
1134  continue;
1135 
1136  VNInfo *SVNI = SR.getVNInfoAt(RegIndex);
1137  assert(SVNI != nullptr && SlotIndex::isSameInstr(SVNI->def, RegIndex));
1138  SR.removeValNo(SVNI);
1139  }
1140  DstLI.removeEmptySubRanges();
1141  } else
1142  LIS->removeVRegDefAt(DstLI, RegIndex);
1143 
1144  // Mark uses as undef.
1145  for (MachineOperand &MO : MRI->reg_nodbg_operands(DstReg)) {
1146  if (MO.isDef() /*|| MO.isUndef()*/)
1147  continue;
1148  const MachineInstr &MI = *MO.getParent();
1149  SlotIndex UseIdx = LIS->getInstructionIndex(&MI);
1150  unsigned UseMask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
1151  bool isLive;
1152  if (UseMask != ~0u && DstLI.hasSubRanges()) {
1153  isLive = false;
1154  for (const LiveInterval::SubRange &SR : DstLI.subranges()) {
1155  if ((SR.LaneMask & UseMask) == 0)
1156  continue;
1157  if (SR.liveAt(UseIdx)) {
1158  isLive = true;
1159  break;
1160  }
1161  }
1162  } else
1163  isLive = DstLI.liveAt(UseIdx);
1164  if (isLive)
1165  continue;
1166  MO.setIsUndef(true);
1167  DEBUG(dbgs() << "\tnew undef: " << UseIdx << '\t' << MI);
1168  }
1169  return true;
1170 }
1171 
1172 void RegisterCoalescer::updateRegDefsUses(unsigned SrcReg,
1173  unsigned DstReg,
1174  unsigned SubIdx) {
1175  bool DstIsPhys = TargetRegisterInfo::isPhysicalRegister(DstReg);
1176  LiveInterval *DstInt = DstIsPhys ? nullptr : &LIS->getInterval(DstReg);
1177 
1180  I = MRI->reg_instr_begin(SrcReg), E = MRI->reg_instr_end();
1181  I != E; ) {
1182  MachineInstr *UseMI = &*(I++);
1183 
1184  // Each instruction can only be rewritten once because sub-register
1185  // composition is not always idempotent. When SrcReg != DstReg, rewriting
1186  // the UseMI operands removes them from the SrcReg use-def chain, but when
1187  // SrcReg is DstReg we could encounter UseMI twice if it has multiple
1188  // operands mentioning the virtual register.
1189  if (SrcReg == DstReg && !Visited.insert(UseMI).second)
1190  continue;
1191 
1193  bool Reads, Writes;
1194  std::tie(Reads, Writes) = UseMI->readsWritesVirtualRegister(SrcReg, &Ops);
1195 
1196  // If SrcReg wasn't read, it may still be the case that DstReg is live-in
1197  // because SrcReg is a sub-register.
1198  if (DstInt && !Reads && SubIdx)
1199  Reads = DstInt->liveAt(LIS->getInstructionIndex(UseMI));
1200 
1201  // Replace SrcReg with DstReg in all UseMI operands.
1202  for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
1203  MachineOperand &MO = UseMI->getOperand(Ops[i]);
1204 
1205  // Adjust <undef> flags in case of sub-register joins. We don't want to
1206  // turn a full def into a read-modify-write sub-register def and vice
1207  // versa.
1208  if (SubIdx && MO.isDef())
1209  MO.setIsUndef(!Reads);
1210 
1211  // A subreg use of a partially undef (super) register may be a complete
1212  // undef use now and then has to be marked that way.
1213  if (SubIdx != 0 && MO.isUse() && MRI->shouldTrackSubRegLiveness(DstReg)) {
1214  if (!DstInt->hasSubRanges()) {
1215  BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator();
1216  unsigned Mask = MRI->getMaxLaneMaskForVReg(DstInt->reg);
1217  DstInt->createSubRangeFrom(Allocator, Mask, *DstInt);
1218  }
1219  unsigned Mask = TRI->getSubRegIndexLaneMask(SubIdx);
1220  bool IsUndef = true;
1221  SlotIndex MIIdx = UseMI->isDebugValue()
1222  ? LIS->getSlotIndexes()->getIndexBefore(UseMI)
1223  : LIS->getInstructionIndex(UseMI);
1224  SlotIndex UseIdx = MIIdx.getRegSlot(true);
1225  for (LiveInterval::SubRange &S : DstInt->subranges()) {
1226  if ((S.LaneMask & Mask) == 0)
1227  continue;
1228  if (S.liveAt(UseIdx)) {
1229  IsUndef = false;
1230  break;
1231  }
1232  }
1233  if (IsUndef) {
1234  MO.setIsUndef(true);
1235  // We found out some subregister use is actually reading an undefined
1236  // value. In some cases the whole vreg has become undefined at this
1237  // point so we have to potentially shrink the main range if the
1238  // use was ending a live segment there.
1239  LiveQueryResult Q = DstInt->Query(MIIdx);
1240  if (Q.valueOut() == nullptr)
1241  ShrinkMainRange = true;
1242  }
1243  }
1244 
1245  if (DstIsPhys)
1246  MO.substPhysReg(DstReg, *TRI);
1247  else
1248  MO.substVirtReg(DstReg, SubIdx, *TRI);
1249  }
1250 
1251  DEBUG({
1252  dbgs() << "\t\tupdated: ";
1253  if (!UseMI->isDebugValue())
1254  dbgs() << LIS->getInstructionIndex(UseMI) << "\t";
1255  dbgs() << *UseMI;
1256  });
1257  }
1258 }
1259 
1260 bool RegisterCoalescer::canJoinPhys(const CoalescerPair &CP) {
1261  // Always join simple intervals that are defined by a single copy from a
1262  // reserved register. This doesn't increase register pressure, so it is
1263  // always beneficial.
1264  if (!MRI->isReserved(CP.getDstReg())) {
1265  DEBUG(dbgs() << "\tCan only merge into reserved registers.\n");
1266  return false;
1267  }
1268 
1269  LiveInterval &JoinVInt = LIS->getInterval(CP.getSrcReg());
1270  if (JoinVInt.containsOneValue())
1271  return true;
1272 
1273  DEBUG(dbgs() << "\tCannot join complex intervals into reserved register.\n");
1274  return false;
1275 }
1276 
1277 bool RegisterCoalescer::joinCopy(MachineInstr *CopyMI, bool &Again) {
1278 
1279  Again = false;
1280  DEBUG(dbgs() << LIS->getInstructionIndex(CopyMI) << '\t' << *CopyMI);
1281 
1282  CoalescerPair CP(*TRI);
1283  if (!CP.setRegisters(CopyMI)) {
1284  DEBUG(dbgs() << "\tNot coalescable.\n");
1285  return false;
1286  }
1287 
1288  if (CP.getNewRC()) {
1289  auto SrcRC = MRI->getRegClass(CP.getSrcReg());
1290  auto DstRC = MRI->getRegClass(CP.getDstReg());
1291  unsigned SrcIdx = CP.getSrcIdx();
1292  unsigned DstIdx = CP.getDstIdx();
1293  if (CP.isFlipped()) {
1294  std::swap(SrcIdx, DstIdx);
1295  std::swap(SrcRC, DstRC);
1296  }
1297  if (!TRI->shouldCoalesce(CopyMI, SrcRC, SrcIdx, DstRC, DstIdx,
1298  CP.getNewRC())) {
1299  DEBUG(dbgs() << "\tSubtarget bailed on coalescing.\n");
1300  return false;
1301  }
1302  }
1303 
1304  // Dead code elimination. This really should be handled by MachineDCE, but
1305  // sometimes dead copies slip through, and we can't generate invalid live
1306  // ranges.
1307  if (!CP.isPhys() && CopyMI->allDefsAreDead()) {
1308  DEBUG(dbgs() << "\tCopy is dead.\n");
1309  DeadDefs.push_back(CopyMI);
1310  eliminateDeadDefs();
1311  return true;
1312  }
1313 
1314  // Eliminate undefs.
1315  if (!CP.isPhys() && eliminateUndefCopy(CopyMI)) {
1316  LIS->RemoveMachineInstrFromMaps(CopyMI);
1317  CopyMI->eraseFromParent();
1318  return false; // Not coalescable.
1319  }
1320 
1321  // Coalesced copies are normally removed immediately, but transformations
1322  // like removeCopyByCommutingDef() can inadvertently create identity copies.
1323  // When that happens, just join the values and remove the copy.
1324  if (CP.getSrcReg() == CP.getDstReg()) {
1325  LiveInterval &LI = LIS->getInterval(CP.getSrcReg());
1326  DEBUG(dbgs() << "\tCopy already coalesced: " << LI << '\n');
1327  const SlotIndex CopyIdx = LIS->getInstructionIndex(CopyMI);
1328  LiveQueryResult LRQ = LI.Query(CopyIdx);
1329  if (VNInfo *DefVNI = LRQ.valueDefined()) {
1330  VNInfo *ReadVNI = LRQ.valueIn();
1331  assert(ReadVNI && "No value before copy and no <undef> flag.");
1332  assert(ReadVNI != DefVNI && "Cannot read and define the same value.");
1333  LI.MergeValueNumberInto(DefVNI, ReadVNI);
1334 
1335  // Process subregister liveranges.
1336  for (LiveInterval::SubRange &S : LI.subranges()) {
1337  LiveQueryResult SLRQ = S.Query(CopyIdx);
1338  if (VNInfo *SDefVNI = SLRQ.valueDefined()) {
1339  VNInfo *SReadVNI = SLRQ.valueIn();
1340  S.MergeValueNumberInto(SDefVNI, SReadVNI);
1341  }
1342  }
1343  DEBUG(dbgs() << "\tMerged values: " << LI << '\n');
1344  }
1345  LIS->RemoveMachineInstrFromMaps(CopyMI);
1346  CopyMI->eraseFromParent();
1347  return true;
1348  }
1349 
1350  // Enforce policies.
1351  if (CP.isPhys()) {
1352  DEBUG(dbgs() << "\tConsidering merging " << PrintReg(CP.getSrcReg(), TRI)
1353  << " with " << PrintReg(CP.getDstReg(), TRI, CP.getSrcIdx())
1354  << '\n');
1355  if (!canJoinPhys(CP)) {
1356  // Before giving up coalescing, if definition of source is defined by
1357  // trivial computation, try rematerializing it.
1358  bool IsDefCopy;
1359  if (reMaterializeTrivialDef(CP, CopyMI, IsDefCopy))
1360  return true;
1361  if (IsDefCopy)
1362  Again = true; // May be possible to coalesce later.
1363  return false;
1364  }
1365  } else {
1366  // When possible, let DstReg be the larger interval.
1367  if (!CP.isPartial() && LIS->getInterval(CP.getSrcReg()).size() >
1368  LIS->getInterval(CP.getDstReg()).size())
1369  CP.flip();
1370 
1371  DEBUG({
1372  dbgs() << "\tConsidering merging to "
1373  << TRI->getRegClassName(CP.getNewRC()) << " with ";
1374  if (CP.getDstIdx() && CP.getSrcIdx())
1375  dbgs() << PrintReg(CP.getDstReg()) << " in "
1376  << TRI->getSubRegIndexName(CP.getDstIdx()) << " and "
1377  << PrintReg(CP.getSrcReg()) << " in "
1378  << TRI->getSubRegIndexName(CP.getSrcIdx()) << '\n';
1379  else
1380  dbgs() << PrintReg(CP.getSrcReg(), TRI) << " in "
1381  << PrintReg(CP.getDstReg(), TRI, CP.getSrcIdx()) << '\n';
1382  });
1383  }
1384 
1385  ShrinkMask = 0;
1386  ShrinkMainRange = false;
1387 
1388  // Okay, attempt to join these two intervals. On failure, this returns false.
1389  // Otherwise, if one of the intervals being joined is a physreg, this method
1390  // always canonicalizes DstInt to be it. The output "SrcInt" will not have
1391  // been modified, so we can use this information below to update aliases.
1392  if (!joinIntervals(CP)) {
1393  // Coalescing failed.
1394 
1395  // If definition of source is defined by trivial computation, try
1396  // rematerializing it.
1397  bool IsDefCopy;
1398  if (reMaterializeTrivialDef(CP, CopyMI, IsDefCopy))
1399  return true;
1400 
1401  // If we can eliminate the copy without merging the live segments, do so
1402  // now.
1403  if (!CP.isPartial() && !CP.isPhys()) {
1404  if (adjustCopiesBackFrom(CP, CopyMI) ||
1405  removeCopyByCommutingDef(CP, CopyMI)) {
1406  LIS->RemoveMachineInstrFromMaps(CopyMI);
1407  CopyMI->eraseFromParent();
1408  DEBUG(dbgs() << "\tTrivial!\n");
1409  return true;
1410  }
1411  }
1412 
1413  // Otherwise, we are unable to join the intervals.
1414  DEBUG(dbgs() << "\tInterference!\n");
1415  Again = true; // May be possible to coalesce later.
1416  return false;
1417  }
1418 
1419  // Coalescing to a virtual register that is of a sub-register class of the
1420  // other. Make sure the resulting register is set to the right register class.
1421  if (CP.isCrossClass()) {
1422  ++numCrossRCs;
1423  MRI->setRegClass(CP.getDstReg(), CP.getNewRC());
1424  }
1425 
1426  // Removing sub-register copies can ease the register class constraints.
1427  // Make sure we attempt to inflate the register class of DstReg.
1428  if (!CP.isPhys() && RegClassInfo.isProperSubClass(CP.getNewRC()))
1429  InflateRegs.push_back(CP.getDstReg());
1430 
1431  // CopyMI has been erased by joinIntervals at this point. Remove it from
1432  // ErasedInstrs since copyCoalesceWorkList() won't add a successful join back
1433  // to the work list. This keeps ErasedInstrs from growing needlessly.
1434  ErasedInstrs.erase(CopyMI);
1435 
1436  // Rewrite all SrcReg operands to DstReg.
1437  // Also update DstReg operands to include DstIdx if it is set.
1438  if (CP.getDstIdx())
1439  updateRegDefsUses(CP.getDstReg(), CP.getDstReg(), CP.getDstIdx());
1440  updateRegDefsUses(CP.getSrcReg(), CP.getDstReg(), CP.getSrcIdx());
1441 
1442  // Shrink subregister ranges if necessary.
1443  if (ShrinkMask != 0) {
1444  LiveInterval &LI = LIS->getInterval(CP.getDstReg());
1445  for (LiveInterval::SubRange &S : LI.subranges()) {
1446  if ((S.LaneMask & ShrinkMask) == 0)
1447  continue;
1448  DEBUG(dbgs() << "Shrink LaneUses (Lane "
1449  << format("%04X", S.LaneMask) << ")\n");
1450  LIS->shrinkToUses(S, LI.reg);
1451  }
1452  LI.removeEmptySubRanges();
1453  }
1454  if (ShrinkMainRange) {
1455  LiveInterval &LI = LIS->getInterval(CP.getDstReg());
1456  shrinkToUses(&LI);
1457  }
1458 
1459  // SrcReg is guaranteed to be the register whose live interval that is
1460  // being merged.
1461  LIS->removeInterval(CP.getSrcReg());
1462 
1463  // Update regalloc hint.
1464  TRI->updateRegAllocHint(CP.getSrcReg(), CP.getDstReg(), *MF);
1465 
1466  DEBUG({
1467  dbgs() << "\tSuccess: " << PrintReg(CP.getSrcReg(), TRI, CP.getSrcIdx())
1468  << " -> " << PrintReg(CP.getDstReg(), TRI, CP.getDstIdx()) << '\n';
1469  dbgs() << "\tResult = ";
1470  if (CP.isPhys())
1471  dbgs() << PrintReg(CP.getDstReg(), TRI);
1472  else
1473  dbgs() << LIS->getInterval(CP.getDstReg());
1474  dbgs() << '\n';
1475  });
1476 
1477  ++numJoins;
1478  return true;
1479 }
1480 
1481 bool RegisterCoalescer::joinReservedPhysReg(CoalescerPair &CP) {
1482  unsigned DstReg = CP.getDstReg();
1483  assert(CP.isPhys() && "Must be a physreg copy");
1484  assert(MRI->isReserved(DstReg) && "Not a reserved register");
1485  LiveInterval &RHS = LIS->getInterval(CP.getSrcReg());
1486  DEBUG(dbgs() << "\t\tRHS = " << RHS << '\n');
1487 
1488  assert(RHS.containsOneValue() && "Invalid join with reserved register");
1489 
1490  // Optimization for reserved registers like ESP. We can only merge with a
1491  // reserved physreg if RHS has a single value that is a copy of DstReg.
1492  // The live range of the reserved register will look like a set of dead defs
1493  // - we don't properly track the live range of reserved registers.
1494 
1495  // Deny any overlapping intervals. This depends on all the reserved
1496  // register live ranges to look like dead defs.
1497  for (MCRegUnitIterator UI(DstReg, TRI); UI.isValid(); ++UI)
1498  if (RHS.overlaps(LIS->getRegUnit(*UI))) {
1499  DEBUG(dbgs() << "\t\tInterference: " << PrintRegUnit(*UI, TRI) << '\n');
1500  return false;
1501  }
1502 
1503  // Skip any value computations, we are not adding new values to the
1504  // reserved register. Also skip merging the live ranges, the reserved
1505  // register live range doesn't need to be accurate as long as all the
1506  // defs are there.
1507 
1508  // Delete the identity copy.
1509  MachineInstr *CopyMI;
1510  if (CP.isFlipped()) {
1511  CopyMI = MRI->getVRegDef(RHS.reg);
1512  } else {
1513  if (!MRI->hasOneNonDBGUse(RHS.reg)) {
1514  DEBUG(dbgs() << "\t\tMultiple vreg uses!\n");
1515  return false;
1516  }
1517 
1518  MachineInstr *DestMI = MRI->getVRegDef(RHS.reg);
1519  CopyMI = &*MRI->use_instr_nodbg_begin(RHS.reg);
1520  const SlotIndex CopyRegIdx = LIS->getInstructionIndex(CopyMI).getRegSlot();
1521  const SlotIndex DestRegIdx = LIS->getInstructionIndex(DestMI).getRegSlot();
1522 
1523  // We checked above that there are no interfering defs of the physical
1524  // register. However, for this case, where we intent to move up the def of
1525  // the physical register, we also need to check for interfering uses.
1526  SlotIndexes *Indexes = LIS->getSlotIndexes();
1527  for (SlotIndex SI = Indexes->getNextNonNullIndex(DestRegIdx);
1528  SI != CopyRegIdx; SI = Indexes->getNextNonNullIndex(SI)) {
1529  MachineInstr *MI = LIS->getInstructionFromIndex(SI);
1530  if (MI->readsRegister(DstReg, TRI)) {
1531  DEBUG(dbgs() << "\t\tInterference (read): " << *MI);
1532  return false;
1533  }
1534 
1535  // We must also check for clobbers caused by regmasks.
1536  for (const auto &MO : MI->operands()) {
1537  if (MO.isRegMask() && MO.clobbersPhysReg(DstReg)) {
1538  DEBUG(dbgs() << "\t\tInterference (regmask clobber): " << *MI);
1539  return false;
1540  }
1541  }
1542  }
1543 
1544  // We're going to remove the copy which defines a physical reserved
1545  // register, so remove its valno, etc.
1546  DEBUG(dbgs() << "\t\tRemoving phys reg def of " << DstReg << " at "
1547  << CopyRegIdx << "\n");
1548 
1549  LIS->removePhysRegDefAt(DstReg, CopyRegIdx);
1550  // Create a new dead def at the new def location.
1551  for (MCRegUnitIterator UI(DstReg, TRI); UI.isValid(); ++UI) {
1552  LiveRange &LR = LIS->getRegUnit(*UI);
1553  LR.createDeadDef(DestRegIdx, LIS->getVNInfoAllocator());
1554  }
1555  }
1556 
1557  LIS->RemoveMachineInstrFromMaps(CopyMI);
1558  CopyMI->eraseFromParent();
1559 
1560  // We don't track kills for reserved registers.
1561  MRI->clearKillFlags(CP.getSrcReg());
1562 
1563  return true;
1564 }
1565 
1566 //===----------------------------------------------------------------------===//
1567 // Interference checking and interval joining
1568 //===----------------------------------------------------------------------===//
1569 //
1570 // In the easiest case, the two live ranges being joined are disjoint, and
1571 // there is no interference to consider. It is quite common, though, to have
1572 // overlapping live ranges, and we need to check if the interference can be
1573 // resolved.
1574 //
1575 // The live range of a single SSA value forms a sub-tree of the dominator tree.
1576 // This means that two SSA values overlap if and only if the def of one value
1577 // is contained in the live range of the other value. As a special case, the
1578 // overlapping values can be defined at the same index.
1579 //
1580 // The interference from an overlapping def can be resolved in these cases:
1581 //
1582 // 1. Coalescable copies. The value is defined by a copy that would become an
1583 // identity copy after joining SrcReg and DstReg. The copy instruction will
1584 // be removed, and the value will be merged with the source value.
1585 //
1586 // There can be several copies back and forth, causing many values to be
1587 // merged into one. We compute a list of ultimate values in the joined live
1588 // range as well as a mappings from the old value numbers.
1589 //
1590 // 2. IMPLICIT_DEF. This instruction is only inserted to ensure all PHI
1591 // predecessors have a live out value. It doesn't cause real interference,
1592 // and can be merged into the value it overlaps. Like a coalescable copy, it
1593 // can be erased after joining.
1594 //
1595 // 3. Copy of external value. The overlapping def may be a copy of a value that
1596 // is already in the other register. This is like a coalescable copy, but
1597 // the live range of the source register must be trimmed after erasing the
1598 // copy instruction:
1599 //
1600 // %src = COPY %ext
1601 // %dst = COPY %ext <-- Remove this COPY, trim the live range of %ext.
1602 //
1603 // 4. Clobbering undefined lanes. Vector registers are sometimes built by
1604 // defining one lane at a time:
1605 //
1606 // %dst:ssub0<def,read-undef> = FOO
1607 // %src = BAR
1608 // %dst:ssub1<def> = COPY %src
1609 //
1610 // The live range of %src overlaps the %dst value defined by FOO, but
1611 // merging %src into %dst:ssub1 is only going to clobber the ssub1 lane
1612 // which was undef anyway.
1613 //
1614 // The value mapping is more complicated in this case. The final live range
1615 // will have different value numbers for both FOO and BAR, but there is no
1616 // simple mapping from old to new values. It may even be necessary to add
1617 // new PHI values.
1618 //
1619 // 5. Clobbering dead lanes. A def may clobber a lane of a vector register that
1620 // is live, but never read. This can happen because we don't compute
1621 // individual live ranges per lane.
1622 //
1623 // %dst<def> = FOO
1624 // %src = BAR
1625 // %dst:ssub1<def> = COPY %src
1626 //
1627 // This kind of interference is only resolved locally. If the clobbered
1628 // lane value escapes the block, the join is aborted.
1629 
1630 namespace {
1631 /// Track information about values in a single virtual register about to be
1632 /// joined. Objects of this class are always created in pairs - one for each
1633 /// side of the CoalescerPair (or one for each lane of a side of the coalescer
1634 /// pair)
1635 class JoinVals {
1636  /// Live range we work on.
1637  LiveRange &LR;
1638  /// (Main) register we work on.
1639  const unsigned Reg;
1640 
1641  /// Reg (and therefore the values in this liverange) will end up as
1642  /// subregister SubIdx in the coalesced register. Either CP.DstIdx or
1643  /// CP.SrcIdx.
1644  const unsigned SubIdx;
1645  /// The LaneMask that this liverange will occupy the coalesced register. May
1646  /// be smaller than the lanemask produced by SubIdx when merging subranges.
1647  const unsigned LaneMask;
1648 
1649  /// This is true when joining sub register ranges, false when joining main
1650  /// ranges.
1651  const bool SubRangeJoin;
1652  /// Whether the current LiveInterval tracks subregister liveness.
1653  const bool TrackSubRegLiveness;
1654 
1655  /// Values that will be present in the final live range.
1656  SmallVectorImpl<VNInfo*> &NewVNInfo;
1657 
1658  const CoalescerPair &CP;
1659  LiveIntervals *LIS;
1660  SlotIndexes *Indexes;
1661  const TargetRegisterInfo *TRI;
1662 
1663  /// Value number assignments. Maps value numbers in LI to entries in
1664  /// NewVNInfo. This is suitable for passing to LiveInterval::join().
1665  SmallVector<int, 8> Assignments;
1666 
1667  /// Conflict resolution for overlapping values.
1668  enum ConflictResolution {
1669  /// No overlap, simply keep this value.
1670  CR_Keep,
1671 
1672  /// Merge this value into OtherVNI and erase the defining instruction.
1673  /// Used for IMPLICIT_DEF, coalescable copies, and copies from external
1674  /// values.
1675  CR_Erase,
1676 
1677  /// Merge this value into OtherVNI but keep the defining instruction.
1678  /// This is for the special case where OtherVNI is defined by the same
1679  /// instruction.
1680  CR_Merge,
1681 
1682  /// Keep this value, and have it replace OtherVNI where possible. This
1683  /// complicates value mapping since OtherVNI maps to two different values
1684  /// before and after this def.
1685  /// Used when clobbering undefined or dead lanes.
1686  CR_Replace,
1687 
1688  /// Unresolved conflict. Visit later when all values have been mapped.
1689  CR_Unresolved,
1690 
1691  /// Unresolvable conflict. Abort the join.
1692  CR_Impossible
1693  };
1694 
1695  /// Per-value info for LI. The lane bit masks are all relative to the final
1696  /// joined register, so they can be compared directly between SrcReg and
1697  /// DstReg.
1698  struct Val {
1699  ConflictResolution Resolution;
1700 
1701  /// Lanes written by this def, 0 for unanalyzed values.
1702  unsigned WriteLanes;
1703 
1704  /// Lanes with defined values in this register. Other lanes are undef and
1705  /// safe to clobber.
1706  unsigned ValidLanes;
1707 
1708  /// Value in LI being redefined by this def.
1709  VNInfo *RedefVNI;
1710 
1711  /// Value in the other live range that overlaps this def, if any.
1712  VNInfo *OtherVNI;
1713 
1714  /// Is this value an IMPLICIT_DEF that can be erased?
1715  ///
1716  /// IMPLICIT_DEF values should only exist at the end of a basic block that
1717  /// is a predecessor to a phi-value. These IMPLICIT_DEF instructions can be
1718  /// safely erased if they are overlapping a live value in the other live
1719  /// interval.
1720  ///
1721  /// Weird control flow graphs and incomplete PHI handling in
1722  /// ProcessImplicitDefs can very rarely create IMPLICIT_DEF values with
1723  /// longer live ranges. Such IMPLICIT_DEF values should be treated like
1724  /// normal values.
1725  bool ErasableImplicitDef;
1726 
1727  /// True when the live range of this value will be pruned because of an
1728  /// overlapping CR_Replace value in the other live range.
1729  bool Pruned;
1730 
1731  /// True once Pruned above has been computed.
1732  bool PrunedComputed;
1733 
1734  Val() : Resolution(CR_Keep), WriteLanes(0), ValidLanes(0),
1735  RedefVNI(nullptr), OtherVNI(nullptr), ErasableImplicitDef(false),
1736  Pruned(false), PrunedComputed(false) {}
1737 
1738  bool isAnalyzed() const { return WriteLanes != 0; }
1739  };
1740 
1741  /// One entry per value number in LI.
1742  SmallVector<Val, 8> Vals;
1743 
1744  /// Compute the bitmask of lanes actually written by DefMI.
1745  /// Set Redef if there are any partial register definitions that depend on the
1746  /// previous value of the register.
1747  unsigned computeWriteLanes(const MachineInstr *DefMI, bool &Redef) const;
1748 
1749  /// Find the ultimate value that VNI was copied from.
1750  std::pair<const VNInfo*,unsigned> followCopyChain(const VNInfo *VNI) const;
1751 
1752  bool valuesIdentical(VNInfo *Val0, VNInfo *Val1, const JoinVals &Other) const;
1753 
1754  /// Analyze ValNo in this live range, and set all fields of Vals[ValNo].
1755  /// Return a conflict resolution when possible, but leave the hard cases as
1756  /// CR_Unresolved.
1757  /// Recursively calls computeAssignment() on this and Other, guaranteeing that
1758  /// both OtherVNI and RedefVNI have been analyzed and mapped before returning.
1759  /// The recursion always goes upwards in the dominator tree, making loops
1760  /// impossible.
1761  ConflictResolution analyzeValue(unsigned ValNo, JoinVals &Other);
1762 
1763  /// Compute the value assignment for ValNo in RI.
1764  /// This may be called recursively by analyzeValue(), but never for a ValNo on
1765  /// the stack.
1766  void computeAssignment(unsigned ValNo, JoinVals &Other);
1767 
1768  /// Assuming ValNo is going to clobber some valid lanes in Other.LR, compute
1769  /// the extent of the tainted lanes in the block.
1770  ///
1771  /// Multiple values in Other.LR can be affected since partial redefinitions
1772  /// can preserve previously tainted lanes.
1773  ///
1774  /// 1 %dst = VLOAD <-- Define all lanes in %dst
1775  /// 2 %src = FOO <-- ValNo to be joined with %dst:ssub0
1776  /// 3 %dst:ssub1 = BAR <-- Partial redef doesn't clear taint in ssub0
1777  /// 4 %dst:ssub0 = COPY %src <-- Conflict resolved, ssub0 wasn't read
1778  ///
1779  /// For each ValNo in Other that is affected, add an (EndIndex, TaintedLanes)
1780  /// entry to TaintedVals.
1781  ///
1782  /// Returns false if the tainted lanes extend beyond the basic block.
1783  bool taintExtent(unsigned, unsigned, JoinVals&,
1784  SmallVectorImpl<std::pair<SlotIndex, unsigned> >&);
1785 
1786  /// Return true if MI uses any of the given Lanes from Reg.
1787  /// This does not include partial redefinitions of Reg.
1788  bool usesLanes(const MachineInstr *MI, unsigned, unsigned, unsigned) const;
1789 
1790  /// Determine if ValNo is a copy of a value number in LR or Other.LR that will
1791  /// be pruned:
1792  ///
1793  /// %dst = COPY %src
1794  /// %src = COPY %dst <-- This value to be pruned.
1795  /// %dst = COPY %src <-- This value is a copy of a pruned value.
1796  bool isPrunedValue(unsigned ValNo, JoinVals &Other);
1797 
1798 public:
1799  JoinVals(LiveRange &LR, unsigned Reg, unsigned SubIdx, unsigned LaneMask,
1800  SmallVectorImpl<VNInfo*> &newVNInfo, const CoalescerPair &cp,
1801  LiveIntervals *lis, const TargetRegisterInfo *TRI, bool SubRangeJoin,
1802  bool TrackSubRegLiveness)
1803  : LR(LR), Reg(Reg), SubIdx(SubIdx), LaneMask(LaneMask),
1804  SubRangeJoin(SubRangeJoin), TrackSubRegLiveness(TrackSubRegLiveness),
1805  NewVNInfo(newVNInfo), CP(cp), LIS(lis), Indexes(LIS->getSlotIndexes()),
1806  TRI(TRI), Assignments(LR.getNumValNums(), -1), Vals(LR.getNumValNums())
1807  {}
1808 
1809  /// Analyze defs in LR and compute a value mapping in NewVNInfo.
1810  /// Returns false if any conflicts were impossible to resolve.
1811  bool mapValues(JoinVals &Other);
1812 
1813  /// Try to resolve conflicts that require all values to be mapped.
1814  /// Returns false if any conflicts were impossible to resolve.
1815  bool resolveConflicts(JoinVals &Other);
1816 
1817  /// Prune the live range of values in Other.LR where they would conflict with
1818  /// CR_Replace values in LR. Collect end points for restoring the live range
1819  /// after joining.
1820  void pruneValues(JoinVals &Other, SmallVectorImpl<SlotIndex> &EndPoints,
1821  bool changeInstrs);
1822 
1823  /// Removes subranges starting at copies that get removed. This sometimes
1824  /// happens when undefined subranges are copied around. These ranges contain
1825  /// no usefull information and can be removed.
1826  void pruneSubRegValues(LiveInterval &LI, unsigned &ShrinkMask);
1827 
1828  /// Erase any machine instructions that have been coalesced away.
1829  /// Add erased instructions to ErasedInstrs.
1830  /// Add foreign virtual registers to ShrinkRegs if their live range ended at
1831  /// the erased instrs.
1832  void eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
1833  SmallVectorImpl<unsigned> &ShrinkRegs);
1834 
1835  /// Remove liverange defs at places where implicit defs will be removed.
1836  void removeImplicitDefs();
1837 
1838  /// Get the value assignments suitable for passing to LiveInterval::join.
1839  const int *getAssignments() const { return Assignments.data(); }
1840 };
1841 } // end anonymous namespace
1842 
1843 unsigned JoinVals::computeWriteLanes(const MachineInstr *DefMI, bool &Redef)
1844  const {
1845  unsigned L = 0;
1846  for (const MachineOperand &MO : DefMI->operands()) {
1847  if (!MO.isReg() || MO.getReg() != Reg || !MO.isDef())
1848  continue;
1849  L |= TRI->getSubRegIndexLaneMask(
1850  TRI->composeSubRegIndices(SubIdx, MO.getSubReg()));
1851  if (MO.readsReg())
1852  Redef = true;
1853  }
1854  return L;
1855 }
1856 
1857 std::pair<const VNInfo*, unsigned> JoinVals::followCopyChain(
1858  const VNInfo *VNI) const {
1859  unsigned Reg = this->Reg;
1860 
1861  while (!VNI->isPHIDef()) {
1862  SlotIndex Def = VNI->def;
1863  MachineInstr *MI = Indexes->getInstructionFromIndex(Def);
1864  assert(MI && "No defining instruction");
1865  if (!MI->isFullCopy())
1866  return std::make_pair(VNI, Reg);
1867  unsigned SrcReg = MI->getOperand(1).getReg();
1869  return std::make_pair(VNI, Reg);
1870 
1871  const LiveInterval &LI = LIS->getInterval(SrcReg);
1872  const VNInfo *ValueIn;
1873  // No subrange involved.
1874  if (!SubRangeJoin || !LI.hasSubRanges()) {
1875  LiveQueryResult LRQ = LI.Query(Def);
1876  ValueIn = LRQ.valueIn();
1877  } else {
1878  // Query subranges. Pick the first matching one.
1879  ValueIn = nullptr;
1880  for (const LiveInterval::SubRange &S : LI.subranges()) {
1881  // Transform lanemask to a mask in the joined live interval.
1882  unsigned SMask = TRI->composeSubRegIndexLaneMask(SubIdx, S.LaneMask);
1883  if ((SMask & LaneMask) == 0)
1884  continue;
1885  LiveQueryResult LRQ = S.Query(Def);
1886  ValueIn = LRQ.valueIn();
1887  break;
1888  }
1889  }
1890  if (ValueIn == nullptr)
1891  break;
1892  VNI = ValueIn;
1893  Reg = SrcReg;
1894  }
1895  return std::make_pair(VNI, Reg);
1896 }
1897 
1898 bool JoinVals::valuesIdentical(VNInfo *Value0, VNInfo *Value1,
1899  const JoinVals &Other) const {
1900  const VNInfo *Orig0;
1901  unsigned Reg0;
1902  std::tie(Orig0, Reg0) = followCopyChain(Value0);
1903  if (Orig0 == Value1)
1904  return true;
1905 
1906  const VNInfo *Orig1;
1907  unsigned Reg1;
1908  std::tie(Orig1, Reg1) = Other.followCopyChain(Value1);
1909 
1910  // The values are equal if they are defined at the same place and use the
1911  // same register. Note that we cannot compare VNInfos directly as some of
1912  // them might be from a copy created in mergeSubRangeInto() while the other
1913  // is from the original LiveInterval.
1914  return Orig0->def == Orig1->def && Reg0 == Reg1;
1915 }
1916 
1917 JoinVals::ConflictResolution
1918 JoinVals::analyzeValue(unsigned ValNo, JoinVals &Other) {
1919  Val &V = Vals[ValNo];
1920  assert(!V.isAnalyzed() && "Value has already been analyzed!");
1921  VNInfo *VNI = LR.getValNumInfo(ValNo);
1922  if (VNI->isUnused()) {
1923  V.WriteLanes = ~0u;
1924  return CR_Keep;
1925  }
1926 
1927  // Get the instruction defining this value, compute the lanes written.
1928  const MachineInstr *DefMI = nullptr;
1929  if (VNI->isPHIDef()) {
1930  // Conservatively assume that all lanes in a PHI are valid.
1931  unsigned Lanes = SubRangeJoin ? 1 : TRI->getSubRegIndexLaneMask(SubIdx);
1932  V.ValidLanes = V.WriteLanes = Lanes;
1933  } else {
1934  DefMI = Indexes->getInstructionFromIndex(VNI->def);
1935  assert(DefMI != nullptr);
1936  if (SubRangeJoin) {
1937  // We don't care about the lanes when joining subregister ranges.
1938  V.WriteLanes = V.ValidLanes = 1;
1939  if (DefMI->isImplicitDef()) {
1940  V.ValidLanes = 0;
1941  V.ErasableImplicitDef = true;
1942  }
1943  } else {
1944  bool Redef = false;
1945  V.ValidLanes = V.WriteLanes = computeWriteLanes(DefMI, Redef);
1946 
1947  // If this is a read-modify-write instruction, there may be more valid
1948  // lanes than the ones written by this instruction.
1949  // This only covers partial redef operands. DefMI may have normal use
1950  // operands reading the register. They don't contribute valid lanes.
1951  //
1952  // This adds ssub1 to the set of valid lanes in %src:
1953  //
1954  // %src:ssub1<def> = FOO
1955  //
1956  // This leaves only ssub1 valid, making any other lanes undef:
1957  //
1958  // %src:ssub1<def,read-undef> = FOO %src:ssub2
1959  //
1960  // The <read-undef> flag on the def operand means that old lane values are
1961  // not important.
1962  if (Redef) {
1963  V.RedefVNI = LR.Query(VNI->def).valueIn();
1964  assert((TrackSubRegLiveness || V.RedefVNI) &&
1965  "Instruction is reading nonexistent value");
1966  if (V.RedefVNI != nullptr) {
1967  computeAssignment(V.RedefVNI->id, Other);
1968  V.ValidLanes |= Vals[V.RedefVNI->id].ValidLanes;
1969  }
1970  }
1971 
1972  // An IMPLICIT_DEF writes undef values.
1973  if (DefMI->isImplicitDef()) {
1974  // We normally expect IMPLICIT_DEF values to be live only until the end
1975  // of their block. If the value is really live longer and gets pruned in
1976  // another block, this flag is cleared again.
1977  V.ErasableImplicitDef = true;
1978  V.ValidLanes &= ~V.WriteLanes;
1979  }
1980  }
1981  }
1982 
1983  // Find the value in Other that overlaps VNI->def, if any.
1984  LiveQueryResult OtherLRQ = Other.LR.Query(VNI->def);
1985 
1986  // It is possible that both values are defined by the same instruction, or
1987  // the values are PHIs defined in the same block. When that happens, the two
1988  // values should be merged into one, but not into any preceding value.
1989  // The first value defined or visited gets CR_Keep, the other gets CR_Merge.
1990  if (VNInfo *OtherVNI = OtherLRQ.valueDefined()) {
1991  assert(SlotIndex::isSameInstr(VNI->def, OtherVNI->def) && "Broken LRQ");
1992 
1993  // One value stays, the other is merged. Keep the earlier one, or the first
1994  // one we see.
1995  if (OtherVNI->def < VNI->def)
1996  Other.computeAssignment(OtherVNI->id, *this);
1997  else if (VNI->def < OtherVNI->def && OtherLRQ.valueIn()) {
1998  // This is an early-clobber def overlapping a live-in value in the other
1999  // register. Not mergeable.
2000  V.OtherVNI = OtherLRQ.valueIn();
2001  return CR_Impossible;
2002  }
2003  V.OtherVNI = OtherVNI;
2004  Val &OtherV = Other.Vals[OtherVNI->id];
2005  // Keep this value, check for conflicts when analyzing OtherVNI.
2006  if (!OtherV.isAnalyzed())
2007  return CR_Keep;
2008  // Both sides have been analyzed now.
2009  // Allow overlapping PHI values. Any real interference would show up in a
2010  // predecessor, the PHI itself can't introduce any conflicts.
2011  if (VNI->isPHIDef())
2012  return CR_Merge;
2013  if (V.ValidLanes & OtherV.ValidLanes)
2014  // Overlapping lanes can't be resolved.
2015  return CR_Impossible;
2016  else
2017  return CR_Merge;
2018  }
2019 
2020  // No simultaneous def. Is Other live at the def?
2021  V.OtherVNI = OtherLRQ.valueIn();
2022  if (!V.OtherVNI)
2023  // No overlap, no conflict.
2024  return CR_Keep;
2025 
2026  assert(!SlotIndex::isSameInstr(VNI->def, V.OtherVNI->def) && "Broken LRQ");
2027 
2028  // We have overlapping values, or possibly a kill of Other.
2029  // Recursively compute assignments up the dominator tree.
2030  Other.computeAssignment(V.OtherVNI->id, *this);
2031  Val &OtherV = Other.Vals[V.OtherVNI->id];
2032 
2033  // Check if OtherV is an IMPLICIT_DEF that extends beyond its basic block.
2034  // This shouldn't normally happen, but ProcessImplicitDefs can leave such
2035  // IMPLICIT_DEF instructions behind, and there is nothing wrong with it
2036  // technically.
2037  //
2038  // WHen it happens, treat that IMPLICIT_DEF as a normal value, and don't try
2039  // to erase the IMPLICIT_DEF instruction.
2040  if (OtherV.ErasableImplicitDef && DefMI &&
2041  DefMI->getParent() != Indexes->getMBBFromIndex(V.OtherVNI->def)) {
2042  DEBUG(dbgs() << "IMPLICIT_DEF defined at " << V.OtherVNI->def
2043  << " extends into BB#" << DefMI->getParent()->getNumber()
2044  << ", keeping it.\n");
2045  OtherV.ErasableImplicitDef = false;
2046  }
2047 
2048  // Allow overlapping PHI values. Any real interference would show up in a
2049  // predecessor, the PHI itself can't introduce any conflicts.
2050  if (VNI->isPHIDef())
2051  return CR_Replace;
2052 
2053  // Check for simple erasable conflicts.
2054  if (DefMI->isImplicitDef()) {
2055  // We need the def for the subregister if there is nothing else live at the
2056  // subrange at this point.
2057  if (TrackSubRegLiveness
2058  && (V.WriteLanes & (OtherV.ValidLanes | OtherV.WriteLanes)) == 0)
2059  return CR_Replace;
2060  return CR_Erase;
2061  }
2062 
2063  // Include the non-conflict where DefMI is a coalescable copy that kills
2064  // OtherVNI. We still want the copy erased and value numbers merged.
2065  if (CP.isCoalescable(DefMI)) {
2066  // Some of the lanes copied from OtherVNI may be undef, making them undef
2067  // here too.
2068  V.ValidLanes &= ~V.WriteLanes | OtherV.ValidLanes;
2069  return CR_Erase;
2070  }
2071 
2072  // This may not be a real conflict if DefMI simply kills Other and defines
2073  // VNI.
2074  if (OtherLRQ.isKill() && OtherLRQ.endPoint() <= VNI->def)
2075  return CR_Keep;
2076 
2077  // Handle the case where VNI and OtherVNI can be proven to be identical:
2078  //
2079  // %other = COPY %ext
2080  // %this = COPY %ext <-- Erase this copy
2081  //
2082  if (DefMI->isFullCopy() && !CP.isPartial()
2083  && valuesIdentical(VNI, V.OtherVNI, Other))
2084  return CR_Erase;
2085 
2086  // If the lanes written by this instruction were all undef in OtherVNI, it is
2087  // still safe to join the live ranges. This can't be done with a simple value
2088  // mapping, though - OtherVNI will map to multiple values:
2089  //
2090  // 1 %dst:ssub0 = FOO <-- OtherVNI
2091  // 2 %src = BAR <-- VNI
2092  // 3 %dst:ssub1 = COPY %src<kill> <-- Eliminate this copy.
2093  // 4 BAZ %dst<kill>
2094  // 5 QUUX %src<kill>
2095  //
2096  // Here OtherVNI will map to itself in [1;2), but to VNI in [2;5). CR_Replace
2097  // handles this complex value mapping.
2098  if ((V.WriteLanes & OtherV.ValidLanes) == 0)
2099  return CR_Replace;
2100 
2101  // If the other live range is killed by DefMI and the live ranges are still
2102  // overlapping, it must be because we're looking at an early clobber def:
2103  //
2104  // %dst<def,early-clobber> = ASM %src<kill>
2105  //
2106  // In this case, it is illegal to merge the two live ranges since the early
2107  // clobber def would clobber %src before it was read.
2108  if (OtherLRQ.isKill()) {
2109  // This case where the def doesn't overlap the kill is handled above.
2110  assert(VNI->def.isEarlyClobber() &&
2111  "Only early clobber defs can overlap a kill");
2112  return CR_Impossible;
2113  }
2114 
2115  // VNI is clobbering live lanes in OtherVNI, but there is still the
2116  // possibility that no instructions actually read the clobbered lanes.
2117  // If we're clobbering all the lanes in OtherVNI, at least one must be read.
2118  // Otherwise Other.RI wouldn't be live here.
2119  if ((TRI->getSubRegIndexLaneMask(Other.SubIdx) & ~V.WriteLanes) == 0)
2120  return CR_Impossible;
2121 
2122  // We need to verify that no instructions are reading the clobbered lanes. To
2123  // save compile time, we'll only check that locally. Don't allow the tainted
2124  // value to escape the basic block.
2125  MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def);
2126  if (OtherLRQ.endPoint() >= Indexes->getMBBEndIdx(MBB))
2127  return CR_Impossible;
2128 
2129  // There are still some things that could go wrong besides clobbered lanes
2130  // being read, for example OtherVNI may be only partially redefined in MBB,
2131  // and some clobbered lanes could escape the block. Save this analysis for
2132  // resolveConflicts() when all values have been mapped. We need to know
2133  // RedefVNI and WriteLanes for any later defs in MBB, and we can't compute
2134  // that now - the recursive analyzeValue() calls must go upwards in the
2135  // dominator tree.
2136  return CR_Unresolved;
2137 }
2138 
2139 void JoinVals::computeAssignment(unsigned ValNo, JoinVals &Other) {
2140  Val &V = Vals[ValNo];
2141  if (V.isAnalyzed()) {
2142  // Recursion should always move up the dominator tree, so ValNo is not
2143  // supposed to reappear before it has been assigned.
2144  assert(Assignments[ValNo] != -1 && "Bad recursion?");
2145  return;
2146  }
2147  switch ((V.Resolution = analyzeValue(ValNo, Other))) {
2148  case CR_Erase:
2149  case CR_Merge:
2150  // Merge this ValNo into OtherVNI.
2151  assert(V.OtherVNI && "OtherVNI not assigned, can't merge.");
2152  assert(Other.Vals[V.OtherVNI->id].isAnalyzed() && "Missing recursion");
2153  Assignments[ValNo] = Other.Assignments[V.OtherVNI->id];
2154  DEBUG(dbgs() << "\t\tmerge " << PrintReg(Reg) << ':' << ValNo << '@'
2155  << LR.getValNumInfo(ValNo)->def << " into "
2156  << PrintReg(Other.Reg) << ':' << V.OtherVNI->id << '@'
2157  << V.OtherVNI->def << " --> @"
2158  << NewVNInfo[Assignments[ValNo]]->def << '\n');
2159  break;
2160  case CR_Replace:
2161  case CR_Unresolved: {
2162  // The other value is going to be pruned if this join is successful.
2163  assert(V.OtherVNI && "OtherVNI not assigned, can't prune");
2164  Val &OtherV = Other.Vals[V.OtherVNI->id];
2165  // We cannot erase an IMPLICIT_DEF if we don't have valid values for all
2166  // its lanes.
2167  if ((OtherV.WriteLanes & ~V.ValidLanes) != 0 && TrackSubRegLiveness)
2168  OtherV.ErasableImplicitDef = false;
2169  OtherV.Pruned = true;
2170  }
2171  // Fall through.
2172  default:
2173  // This value number needs to go in the final joined live range.
2174  Assignments[ValNo] = NewVNInfo.size();
2175  NewVNInfo.push_back(LR.getValNumInfo(ValNo));
2176  break;
2177  }
2178 }
2179 
2180 bool JoinVals::mapValues(JoinVals &Other) {
2181  for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
2182  computeAssignment(i, Other);
2183  if (Vals[i].Resolution == CR_Impossible) {
2184  DEBUG(dbgs() << "\t\tinterference at " << PrintReg(Reg) << ':' << i
2185  << '@' << LR.getValNumInfo(i)->def << '\n');
2186  return false;
2187  }
2188  }
2189  return true;
2190 }
2191 
2192 bool JoinVals::
2193 taintExtent(unsigned ValNo, unsigned TaintedLanes, JoinVals &Other,
2194  SmallVectorImpl<std::pair<SlotIndex, unsigned> > &TaintExtent) {
2195  VNInfo *VNI = LR.getValNumInfo(ValNo);
2196  MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def);
2197  SlotIndex MBBEnd = Indexes->getMBBEndIdx(MBB);
2198 
2199  // Scan Other.LR from VNI.def to MBBEnd.
2200  LiveInterval::iterator OtherI = Other.LR.find(VNI->def);
2201  assert(OtherI != Other.LR.end() && "No conflict?");
2202  do {
2203  // OtherI is pointing to a tainted value. Abort the join if the tainted
2204  // lanes escape the block.
2205  SlotIndex End = OtherI->end;
2206  if (End >= MBBEnd) {
2207  DEBUG(dbgs() << "\t\ttaints global " << PrintReg(Other.Reg) << ':'
2208  << OtherI->valno->id << '@' << OtherI->start << '\n');
2209  return false;
2210  }
2211  DEBUG(dbgs() << "\t\ttaints local " << PrintReg(Other.Reg) << ':'
2212  << OtherI->valno->id << '@' << OtherI->start
2213  << " to " << End << '\n');
2214  // A dead def is not a problem.
2215  if (End.isDead())
2216  break;
2217  TaintExtent.push_back(std::make_pair(End, TaintedLanes));
2218 
2219  // Check for another def in the MBB.
2220  if (++OtherI == Other.LR.end() || OtherI->start >= MBBEnd)
2221  break;
2222 
2223  // Lanes written by the new def are no longer tainted.
2224  const Val &OV = Other.Vals[OtherI->valno->id];
2225  TaintedLanes &= ~OV.WriteLanes;
2226  if (!OV.RedefVNI)
2227  break;
2228  } while (TaintedLanes);
2229  return true;
2230 }
2231 
2232 bool JoinVals::usesLanes(const MachineInstr *MI, unsigned Reg, unsigned SubIdx,
2233  unsigned Lanes) const {
2234  if (MI->isDebugValue())
2235  return false;
2236  for (const MachineOperand &MO : MI->operands()) {
2237  if (!MO.isReg() || MO.isDef() || MO.getReg() != Reg)
2238  continue;
2239  if (!MO.readsReg())
2240  continue;
2241  if (Lanes & TRI->getSubRegIndexLaneMask(
2242  TRI->composeSubRegIndices(SubIdx, MO.getSubReg())))
2243  return true;
2244  }
2245  return false;
2246 }
2247 
2248 bool JoinVals::resolveConflicts(JoinVals &Other) {
2249  for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
2250  Val &V = Vals[i];
2251  assert (V.Resolution != CR_Impossible && "Unresolvable conflict");
2252  if (V.Resolution != CR_Unresolved)
2253  continue;
2254  DEBUG(dbgs() << "\t\tconflict at " << PrintReg(Reg) << ':' << i
2255  << '@' << LR.getValNumInfo(i)->def << '\n');
2256  if (SubRangeJoin)
2257  return false;
2258 
2259  ++NumLaneConflicts;
2260  assert(V.OtherVNI && "Inconsistent conflict resolution.");
2261  VNInfo *VNI = LR.getValNumInfo(i);
2262  const Val &OtherV = Other.Vals[V.OtherVNI->id];
2263 
2264  // VNI is known to clobber some lanes in OtherVNI. If we go ahead with the
2265  // join, those lanes will be tainted with a wrong value. Get the extent of
2266  // the tainted lanes.
2267  unsigned TaintedLanes = V.WriteLanes & OtherV.ValidLanes;
2269  if (!taintExtent(i, TaintedLanes, Other, TaintExtent))
2270  // Tainted lanes would extend beyond the basic block.
2271  return false;
2272 
2273  assert(!TaintExtent.empty() && "There should be at least one conflict.");
2274 
2275  // Now look at the instructions from VNI->def to TaintExtent (inclusive).
2276  MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def);
2277  MachineBasicBlock::iterator MI = MBB->begin();
2278  if (!VNI->isPHIDef()) {
2279  MI = Indexes->getInstructionFromIndex(VNI->def);
2280  // No need to check the instruction defining VNI for reads.
2281  ++MI;
2282  }
2283  assert(!SlotIndex::isSameInstr(VNI->def, TaintExtent.front().first) &&
2284  "Interference ends on VNI->def. Should have been handled earlier");
2285  MachineInstr *LastMI =
2286  Indexes->getInstructionFromIndex(TaintExtent.front().first);
2287  assert(LastMI && "Range must end at a proper instruction");
2288  unsigned TaintNum = 0;
2289  for(;;) {
2290  assert(MI != MBB->end() && "Bad LastMI");
2291  if (usesLanes(MI, Other.Reg, Other.SubIdx, TaintedLanes)) {
2292  DEBUG(dbgs() << "\t\ttainted lanes used by: " << *MI);
2293  return false;
2294  }
2295  // LastMI is the last instruction to use the current value.
2296  if (&*MI == LastMI) {
2297  if (++TaintNum == TaintExtent.size())
2298  break;
2299  LastMI = Indexes->getInstructionFromIndex(TaintExtent[TaintNum].first);
2300  assert(LastMI && "Range must end at a proper instruction");
2301  TaintedLanes = TaintExtent[TaintNum].second;
2302  }
2303  ++MI;
2304  }
2305 
2306  // The tainted lanes are unused.
2307  V.Resolution = CR_Replace;
2308  ++NumLaneResolves;
2309  }
2310  return true;
2311 }
2312 
2313 bool JoinVals::isPrunedValue(unsigned ValNo, JoinVals &Other) {
2314  Val &V = Vals[ValNo];
2315  if (V.Pruned || V.PrunedComputed)
2316  return V.Pruned;
2317 
2318  if (V.Resolution != CR_Erase && V.Resolution != CR_Merge)
2319  return V.Pruned;
2320 
2321  // Follow copies up the dominator tree and check if any intermediate value
2322  // has been pruned.
2323  V.PrunedComputed = true;
2324  V.Pruned = Other.isPrunedValue(V.OtherVNI->id, *this);
2325  return V.Pruned;
2326 }
2327 
2328 void JoinVals::pruneValues(JoinVals &Other,
2329  SmallVectorImpl<SlotIndex> &EndPoints,
2330  bool changeInstrs) {
2331  for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
2332  SlotIndex Def = LR.getValNumInfo(i)->def;
2333  switch (Vals[i].Resolution) {
2334  case CR_Keep:
2335  break;
2336  case CR_Replace: {
2337  // This value takes precedence over the value in Other.LR.
2338  LIS->pruneValue(Other.LR, Def, &EndPoints);
2339  // Check if we're replacing an IMPLICIT_DEF value. The IMPLICIT_DEF
2340  // instructions are only inserted to provide a live-out value for PHI
2341  // predecessors, so the instruction should simply go away once its value
2342  // has been replaced.
2343  Val &OtherV = Other.Vals[Vals[i].OtherVNI->id];
2344  bool EraseImpDef = OtherV.ErasableImplicitDef &&
2345  OtherV.Resolution == CR_Keep;
2346  if (!Def.isBlock()) {
2347  if (changeInstrs) {
2348  // Remove <def,read-undef> flags. This def is now a partial redef.
2349  // Also remove <def,dead> flags since the joined live range will
2350  // continue past this instruction.
2351  for (MachineOperand &MO :
2352  Indexes->getInstructionFromIndex(Def)->operands()) {
2353  if (MO.isReg() && MO.isDef() && MO.getReg() == Reg) {
2354  MO.setIsUndef(EraseImpDef);
2355  MO.setIsDead(false);
2356  }
2357  }
2358  }
2359  // This value will reach instructions below, but we need to make sure
2360  // the live range also reaches the instruction at Def.
2361  if (!EraseImpDef)
2362  EndPoints.push_back(Def);
2363  }
2364  DEBUG(dbgs() << "\t\tpruned " << PrintReg(Other.Reg) << " at " << Def
2365  << ": " << Other.LR << '\n');
2366  break;
2367  }
2368  case CR_Erase:
2369  case CR_Merge:
2370  if (isPrunedValue(i, Other)) {
2371  // This value is ultimately a copy of a pruned value in LR or Other.LR.
2372  // We can no longer trust the value mapping computed by
2373  // computeAssignment(), the value that was originally copied could have
2374  // been replaced.
2375  LIS->pruneValue(LR, Def, &EndPoints);
2376  DEBUG(dbgs() << "\t\tpruned all of " << PrintReg(Reg) << " at "
2377  << Def << ": " << LR << '\n');
2378  }
2379  break;
2380  case CR_Unresolved:
2381  case CR_Impossible:
2382  llvm_unreachable("Unresolved conflicts");
2383  }
2384  }
2385 }
2386 
2387 void JoinVals::pruneSubRegValues(LiveInterval &LI, unsigned &ShrinkMask)
2388 {
2389  // Look for values being erased.
2390  bool DidPrune = false;
2391  for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
2392  if (Vals[i].Resolution != CR_Erase)
2393  continue;
2394 
2395  // Check subranges at the point where the copy will be removed.
2396  SlotIndex Def = LR.getValNumInfo(i)->def;
2397  for (LiveInterval::SubRange &S : LI.subranges()) {
2398  LiveQueryResult Q = S.Query(Def);
2399 
2400  // If a subrange starts at the copy then an undefined value has been
2401  // copied and we must remove that subrange value as well.
2402  VNInfo *ValueOut = Q.valueOutOrDead();
2403  if (ValueOut != nullptr && Q.valueIn() == nullptr) {
2404  DEBUG(dbgs() << "\t\tPrune sublane " << format("%04X", S.LaneMask)
2405  << " at " << Def << "\n");
2406  LIS->pruneValue(S, Def, nullptr);
2407  DidPrune = true;
2408  // Mark value number as unused.
2409  ValueOut->markUnused();
2410  continue;
2411  }
2412  // If a subrange ends at the copy, then a value was copied but only
2413  // partially used later. Shrink the subregister range apropriately.
2414  if (Q.valueIn() != nullptr && Q.valueOut() == nullptr) {
2415  DEBUG(dbgs() << "\t\tDead uses at sublane "
2416  << format("%04X", S.LaneMask) << " at " << Def << "\n");
2417  ShrinkMask |= S.LaneMask;
2418  }
2419  }
2420  }
2421  if (DidPrune)
2422  LI.removeEmptySubRanges();
2423 }
2424 
2425 void JoinVals::removeImplicitDefs() {
2426  for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
2427  Val &V = Vals[i];
2428  if (V.Resolution != CR_Keep || !V.ErasableImplicitDef || !V.Pruned)
2429  continue;
2430 
2431  VNInfo *VNI = LR.getValNumInfo(i);
2432  VNI->markUnused();
2433  LR.removeValNo(VNI);
2434  }
2435 }
2436 
2437 void JoinVals::eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
2438  SmallVectorImpl<unsigned> &ShrinkRegs) {
2439  for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
2440  // Get the def location before markUnused() below invalidates it.
2441  SlotIndex Def = LR.getValNumInfo(i)->def;
2442  switch (Vals[i].Resolution) {
2443  case CR_Keep: {
2444  // If an IMPLICIT_DEF value is pruned, it doesn't serve a purpose any
2445  // longer. The IMPLICIT_DEF instructions are only inserted by
2446  // PHIElimination to guarantee that all PHI predecessors have a value.
2447  if (!Vals[i].ErasableImplicitDef || !Vals[i].Pruned)
2448  break;
2449  // Remove value number i from LR.
2450  VNInfo *VNI = LR.getValNumInfo(i);
2451  LR.removeValNo(VNI);
2452  // Note that this VNInfo is reused and still referenced in NewVNInfo,
2453  // make it appear like an unused value number.
2454  VNI->markUnused();
2455  DEBUG(dbgs() << "\t\tremoved " << i << '@' << Def << ": " << LR << '\n');
2456  // FALL THROUGH.
2457  }
2458 
2459  case CR_Erase: {
2460  MachineInstr *MI = Indexes->getInstructionFromIndex(Def);
2461  assert(MI && "No instruction to erase");
2462  if (MI->isCopy()) {
2463  unsigned Reg = MI->getOperand(1).getReg();
2465  Reg != CP.getSrcReg() && Reg != CP.getDstReg())
2466  ShrinkRegs.push_back(Reg);
2467  }
2468  ErasedInstrs.insert(MI);
2469  DEBUG(dbgs() << "\t\terased:\t" << Def << '\t' << *MI);
2470  LIS->RemoveMachineInstrFromMaps(MI);
2471  MI->eraseFromParent();
2472  break;
2473  }
2474  default:
2475  break;
2476  }
2477  }
2478 }
2479 
2480 bool RegisterCoalescer::joinSubRegRanges(LiveRange &LRange, LiveRange &RRange,
2481  unsigned LaneMask,
2482  const CoalescerPair &CP) {
2483  SmallVector<VNInfo*, 16> NewVNInfo;
2484  JoinVals RHSVals(RRange, CP.getSrcReg(), CP.getSrcIdx(), LaneMask,
2485  NewVNInfo, CP, LIS, TRI, true, true);
2486  JoinVals LHSVals(LRange, CP.getDstReg(), CP.getDstIdx(), LaneMask,
2487  NewVNInfo, CP, LIS, TRI, true, true);
2488 
2489  // Compute NewVNInfo and resolve conflicts (see also joinVirtRegs())
2490  // We should be able to resolve all conflicts here as we could successfully do
2491  // it on the mainrange already. There is however a problem when multiple
2492  // ranges get mapped to the "overflow" lane mask bit which creates unexpected
2493  // interferences.
2494  if (!LHSVals.mapValues(RHSVals) || !RHSVals.mapValues(LHSVals)) {
2495  DEBUG(dbgs() << "*** Couldn't join subrange!\n");
2496  return false;
2497  }
2498  if (!LHSVals.resolveConflicts(RHSVals) ||
2499  !RHSVals.resolveConflicts(LHSVals)) {
2500  DEBUG(dbgs() << "*** Couldn't join subrange!\n");
2501  return false;
2502  }
2503 
2504  // The merging algorithm in LiveInterval::join() can't handle conflicting
2505  // value mappings, so we need to remove any live ranges that overlap a
2506  // CR_Replace resolution. Collect a set of end points that can be used to
2507  // restore the live range after joining.
2508  SmallVector<SlotIndex, 8> EndPoints;
2509  LHSVals.pruneValues(RHSVals, EndPoints, false);
2510  RHSVals.pruneValues(LHSVals, EndPoints, false);
2511 
2512  LHSVals.removeImplicitDefs();
2513  RHSVals.removeImplicitDefs();
2514 
2515  LRange.verify();
2516  RRange.verify();
2517 
2518  // Join RRange into LHS.
2519  LRange.join(RRange, LHSVals.getAssignments(), RHSVals.getAssignments(),
2520  NewVNInfo);
2521 
2522  DEBUG(dbgs() << "\t\tjoined lanes: " << LRange << "\n");
2523  if (EndPoints.empty())
2524  return true;
2525 
2526  // Recompute the parts of the live range we had to remove because of
2527  // CR_Replace conflicts.
2528  DEBUG(dbgs() << "\t\trestoring liveness to " << EndPoints.size()
2529  << " points: " << LRange << '\n');
2530  LIS->extendToIndices(LRange, EndPoints);
2531  return true;
2532 }
2533 
2534 bool RegisterCoalescer::mergeSubRangeInto(LiveInterval &LI,
2535  const LiveRange &ToMerge,
2536  unsigned LaneMask, CoalescerPair &CP) {
2537  BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator();
2538  for (LiveInterval::SubRange &R : LI.subranges()) {
2539  unsigned RMask = R.LaneMask;
2540  // LaneMask of subregisters common to subrange R and ToMerge.
2541  unsigned Common = RMask & LaneMask;
2542  // There is nothing to do without common subregs.
2543  if (Common == 0)
2544  continue;
2545 
2546  DEBUG(dbgs() << format("\t\tCopy+Merge %04X into %04X\n", RMask, Common));
2547  // LaneMask of subregisters contained in the R range but not in ToMerge,
2548  // they have to split into their own subrange.
2549  unsigned LRest = RMask & ~LaneMask;
2550  LiveInterval::SubRange *CommonRange;
2551  if (LRest != 0) {
2552  R.LaneMask = LRest;
2553  DEBUG(dbgs() << format("\t\tReduce Lane to %04X\n", LRest));
2554  // Duplicate SubRange for newly merged common stuff.
2555  CommonRange = LI.createSubRangeFrom(Allocator, Common, R);
2556  } else {
2557  // Reuse the existing range.
2558  R.LaneMask = Common;
2559  CommonRange = &R;
2560  }
2561  LiveRange RangeCopy(ToMerge, Allocator);
2562  if (!joinSubRegRanges(*CommonRange, RangeCopy, Common, CP))
2563  return false;
2564  LaneMask &= ~RMask;
2565  }
2566 
2567  if (LaneMask != 0) {
2568  DEBUG(dbgs() << format("\t\tNew Lane %04X\n", LaneMask));
2569  LI.createSubRangeFrom(Allocator, LaneMask, ToMerge);
2570  }
2571  return true;
2572 }
2573 
2574 bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) {
2575  SmallVector<VNInfo*, 16> NewVNInfo;
2576  LiveInterval &RHS = LIS->getInterval(CP.getSrcReg());
2577  LiveInterval &LHS = LIS->getInterval(CP.getDstReg());
2578  bool TrackSubRegLiveness = MRI->shouldTrackSubRegLiveness(*CP.getNewRC());
2579  JoinVals RHSVals(RHS, CP.getSrcReg(), CP.getSrcIdx(), 0, NewVNInfo, CP, LIS,
2580  TRI, false, TrackSubRegLiveness);
2581  JoinVals LHSVals(LHS, CP.getDstReg(), CP.getDstIdx(), 0, NewVNInfo, CP, LIS,
2582  TRI, false, TrackSubRegLiveness);
2583 
2584  DEBUG(dbgs() << "\t\tRHS = " << RHS
2585  << "\n\t\tLHS = " << LHS
2586  << '\n');
2587 
2588  // First compute NewVNInfo and the simple value mappings.
2589  // Detect impossible conflicts early.
2590  if (!LHSVals.mapValues(RHSVals) || !RHSVals.mapValues(LHSVals))
2591  return false;
2592 
2593  // Some conflicts can only be resolved after all values have been mapped.
2594  if (!LHSVals.resolveConflicts(RHSVals) || !RHSVals.resolveConflicts(LHSVals))
2595  return false;
2596 
2597  // All clear, the live ranges can be merged.
2598  if (RHS.hasSubRanges() || LHS.hasSubRanges()) {
2599  BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator();
2600 
2601  // Transform lanemasks from the LHS to masks in the coalesced register and
2602  // create initial subranges if necessary.
2603  unsigned DstIdx = CP.getDstIdx();
2604  if (!LHS.hasSubRanges()) {
2605  unsigned Mask = DstIdx == 0 ? CP.getNewRC()->getLaneMask()
2606  : TRI->getSubRegIndexLaneMask(DstIdx);
2607  // LHS must support subregs or we wouldn't be in this codepath.
2608  assert(Mask != 0);
2609  LHS.createSubRangeFrom(Allocator, Mask, LHS);
2610  } else if (DstIdx != 0) {
2611  // Transform LHS lanemasks to new register class if necessary.
2612  for (LiveInterval::SubRange &R : LHS.subranges()) {
2613  unsigned Mask = TRI->composeSubRegIndexLaneMask(DstIdx, R.LaneMask);
2614  R.LaneMask = Mask;
2615  }
2616  }
2617  DEBUG(dbgs() << "\t\tLHST = " << PrintReg(CP.getDstReg())
2618  << ' ' << LHS << '\n');
2619 
2620  // Determine lanemasks of RHS in the coalesced register and merge subranges.
2621  unsigned SrcIdx = CP.getSrcIdx();
2622  bool Abort = false;
2623  if (!RHS.hasSubRanges()) {
2624  unsigned Mask = SrcIdx == 0 ? CP.getNewRC()->getLaneMask()
2625  : TRI->getSubRegIndexLaneMask(SrcIdx);
2626  if (!mergeSubRangeInto(LHS, RHS, Mask, CP))
2627  Abort = true;
2628  } else {
2629  // Pair up subranges and merge.
2630  for (LiveInterval::SubRange &R : RHS.subranges()) {
2631  unsigned Mask = TRI->composeSubRegIndexLaneMask(SrcIdx, R.LaneMask);
2632  if (!mergeSubRangeInto(LHS, R, Mask, CP)) {
2633  Abort = true;
2634  break;
2635  }
2636  }
2637  }
2638  if (Abort) {
2639  // This shouldn't have happened :-(
2640  // However we are aware of at least one existing problem where we
2641  // can't merge subranges when multiple ranges end up in the
2642  // "overflow bit" 32. As a workaround we drop all subregister ranges
2643  // which means we loose some precision but are back to a well defined
2644  // state.
2646  CP.getNewRC()->getLaneMask())
2647  && "SubRange merge should only fail when merging into bit 32.");
2648  DEBUG(dbgs() << "\tSubrange join aborted!\n");
2649  LHS.clearSubRanges();
2650  RHS.clearSubRanges();
2651  } else {
2652  DEBUG(dbgs() << "\tJoined SubRanges " << LHS << "\n");
2653 
2654  LHSVals.pruneSubRegValues(LHS, ShrinkMask);
2655  RHSVals.pruneSubRegValues(LHS, ShrinkMask);
2656  }
2657  }
2658 
2659  // The merging algorithm in LiveInterval::join() can't handle conflicting
2660  // value mappings, so we need to remove any live ranges that overlap a
2661  // CR_Replace resolution. Collect a set of end points that can be used to
2662  // restore the live range after joining.
2663  SmallVector<SlotIndex, 8> EndPoints;
2664  LHSVals.pruneValues(RHSVals, EndPoints, true);
2665  RHSVals.pruneValues(LHSVals, EndPoints, true);
2666 
2667  // Erase COPY and IMPLICIT_DEF instructions. This may cause some external
2668  // registers to require trimming.
2669  SmallVector<unsigned, 8> ShrinkRegs;
2670  LHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs);
2671  RHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs);
2672  while (!ShrinkRegs.empty())
2673  shrinkToUses(&LIS->getInterval(ShrinkRegs.pop_back_val()));
2674 
2675  // Join RHS into LHS.
2676  LHS.join(RHS, LHSVals.getAssignments(), RHSVals.getAssignments(), NewVNInfo);
2677 
2678  // Kill flags are going to be wrong if the live ranges were overlapping.
2679  // Eventually, we should simply clear all kill flags when computing live
2680  // ranges. They are reinserted after register allocation.
2681  MRI->clearKillFlags(LHS.reg);
2682  MRI->clearKillFlags(RHS.reg);
2683 
2684  if (!EndPoints.empty()) {
2685  // Recompute the parts of the live range we had to remove because of
2686  // CR_Replace conflicts.
2687  DEBUG(dbgs() << "\t\trestoring liveness to " << EndPoints.size()
2688  << " points: " << LHS << '\n');
2689  LIS->extendToIndices((LiveRange&)LHS, EndPoints);
2690  }
2691 
2692  return true;
2693 }
2694 
2695 bool RegisterCoalescer::joinIntervals(CoalescerPair &CP) {
2696  return CP.isPhys() ? joinReservedPhysReg(CP) : joinVirtRegs(CP);
2697 }
2698 
2699 namespace {
2700 /// Information concerning MBB coalescing priority.
2701 struct MBBPriorityInfo {
2702  MachineBasicBlock *MBB;
2703  unsigned Depth;
2704  bool IsSplit;
2705 
2706  MBBPriorityInfo(MachineBasicBlock *mbb, unsigned depth, bool issplit)
2707  : MBB(mbb), Depth(depth), IsSplit(issplit) {}
2708 };
2709 }
2710 
2711 /// C-style comparator that sorts first based on the loop depth of the basic
2712 /// block (the unsigned), and then on the MBB number.
2713 ///
2714 /// EnableGlobalCopies assumes that the primary sort key is loop depth.
2715 static int compareMBBPriority(const MBBPriorityInfo *LHS,
2716  const MBBPriorityInfo *RHS) {
2717  // Deeper loops first
2718  if (LHS->Depth != RHS->Depth)
2719  return LHS->Depth > RHS->Depth ? -1 : 1;
2720 
2721  // Try to unsplit critical edges next.
2722  if (LHS->IsSplit != RHS->IsSplit)
2723  return LHS->IsSplit ? -1 : 1;
2724 
2725  // Prefer blocks that are more connected in the CFG. This takes care of
2726  // the most difficult copies first while intervals are short.
2727  unsigned cl = LHS->MBB->pred_size() + LHS->MBB->succ_size();
2728  unsigned cr = RHS->MBB->pred_size() + RHS->MBB->succ_size();
2729  if (cl != cr)
2730  return cl > cr ? -1 : 1;
2731 
2732  // As a last resort, sort by block number.
2733  return LHS->MBB->getNumber() < RHS->MBB->getNumber() ? -1 : 1;
2734 }
2735 
2736 /// \returns true if the given copy uses or defines a local live range.
2737 static bool isLocalCopy(MachineInstr *Copy, const LiveIntervals *LIS) {
2738  if (!Copy->isCopy())
2739  return false;
2740 
2741  if (Copy->getOperand(1).isUndef())
2742  return false;
2743 
2744  unsigned SrcReg = Copy->getOperand(1).getReg();
2745  unsigned DstReg = Copy->getOperand(0).getReg();
2748  return false;
2749 
2750  return LIS->intervalIsInOneMBB(LIS->getInterval(SrcReg))
2751  || LIS->intervalIsInOneMBB(LIS->getInterval(DstReg));
2752 }
2753 
2754 bool RegisterCoalescer::
2755 copyCoalesceWorkList(MutableArrayRef<MachineInstr*> CurrList) {
2756  bool Progress = false;
2757  for (unsigned i = 0, e = CurrList.size(); i != e; ++i) {
2758  if (!CurrList[i])
2759  continue;
2760  // Skip instruction pointers that have already been erased, for example by
2761  // dead code elimination.
2762  if (ErasedInstrs.erase(CurrList[i])) {
2763  CurrList[i] = nullptr;
2764  continue;
2765  }
2766  bool Again = false;
2767  bool Success = joinCopy(CurrList[i], Again);
2768  Progress |= Success;
2769  if (Success || !Again)
2770  CurrList[i] = nullptr;
2771  }
2772  return Progress;
2773 }
2774 
2775 /// Check if DstReg is a terminal node.
2776 /// I.e., it does not have any affinity other than \p Copy.
2777 static bool isTerminalReg(unsigned DstReg, const MachineInstr &Copy,
2778  const MachineRegisterInfo *MRI) {
2779  assert(Copy.isCopyLike());
2780  // Check if the destination of this copy as any other affinity.
2781  for (const MachineInstr &MI : MRI->reg_nodbg_instructions(DstReg))
2782  if (&MI != &Copy && MI.isCopyLike())
2783  return false;
2784  return true;
2785 }
2786 
2787 bool RegisterCoalescer::applyTerminalRule(const MachineInstr &Copy) const {
2788  assert(Copy.isCopyLike());
2789  if (!UseTerminalRule)
2790  return false;
2791  unsigned DstReg, DstSubReg, SrcReg, SrcSubReg;
2792  isMoveInstr(*TRI, &Copy, SrcReg, DstReg, SrcSubReg, DstSubReg);
2793  // Check if the destination of this copy has any other affinity.
2795  // If SrcReg is a physical register, the copy won't be coalesced.
2796  // Ignoring it may have other side effect (like missing
2797  // rematerialization). So keep it.
2799  !isTerminalReg(DstReg, Copy, MRI))
2800  return false;
2801 
2802  // DstReg is a terminal node. Check if it inteferes with any other
2803  // copy involving SrcReg.
2804  const MachineBasicBlock *OrigBB = Copy.getParent();
2805  const LiveInterval &DstLI = LIS->getInterval(DstReg);
2806  for (const MachineInstr &MI : MRI->reg_nodbg_instructions(SrcReg)) {
2807  // Technically we should check if the weight of the new copy is
2808  // interesting compared to the other one and update the weight
2809  // of the copies accordingly. However, this would only work if
2810  // we would gather all the copies first then coalesce, whereas
2811  // right now we interleave both actions.
2812  // For now, just consider the copies that are in the same block.
2813  if (&MI == &Copy || !MI.isCopyLike() || MI.getParent() != OrigBB)
2814  continue;
2815  unsigned OtherReg, OtherSubReg, OtherSrcReg, OtherSrcSubReg;
2816  isMoveInstr(*TRI, &Copy, OtherSrcReg, OtherReg, OtherSrcSubReg,
2817  OtherSubReg);
2818  if (OtherReg == SrcReg)
2819  OtherReg = OtherSrcReg;
2820  // Check if OtherReg is a non-terminal.
2822  isTerminalReg(OtherReg, MI, MRI))
2823  continue;
2824  // Check that OtherReg interfere with DstReg.
2825  if (LIS->getInterval(OtherReg).overlaps(DstLI)) {
2826  DEBUG(dbgs() << "Apply terminal rule for: " << PrintReg(DstReg) << '\n');
2827  return true;
2828  }
2829  }
2830  return false;
2831 }
2832 
2833 void
2834 RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) {
2835  DEBUG(dbgs() << MBB->getName() << ":\n");
2836 
2837  // Collect all copy-like instructions in MBB. Don't start coalescing anything
2838  // yet, it might invalidate the iterator.
2839  const unsigned PrevSize = WorkList.size();
2840  if (JoinGlobalCopies) {
2841  SmallVector<MachineInstr*, 2> LocalTerminals;
2842  SmallVector<MachineInstr*, 2> GlobalTerminals;
2843  // Coalesce copies bottom-up to coalesce local defs before local uses. They
2844  // are not inherently easier to resolve, but slightly preferable until we
2845  // have local live range splitting. In particular this is required by
2846  // cmp+jmp macro fusion.
2847  for (MachineBasicBlock::iterator MII = MBB->begin(), E = MBB->end();
2848  MII != E; ++MII) {
2849  if (!MII->isCopyLike())
2850  continue;
2851  bool ApplyTerminalRule = applyTerminalRule(*MII);
2852  if (isLocalCopy(&(*MII), LIS)) {
2853  if (ApplyTerminalRule)
2854  LocalTerminals.push_back(&(*MII));
2855  else
2856  LocalWorkList.push_back(&(*MII));
2857  } else {
2858  if (ApplyTerminalRule)
2859  GlobalTerminals.push_back(&(*MII));
2860  else
2861  WorkList.push_back(&(*MII));
2862  }
2863  }
2864  // Append the copies evicted by the terminal rule at the end of the list.
2865  LocalWorkList.append(LocalTerminals.begin(), LocalTerminals.end());
2866  WorkList.append(GlobalTerminals.begin(), GlobalTerminals.end());
2867  }
2868  else {
2870  for (MachineBasicBlock::iterator MII = MBB->begin(), E = MBB->end();
2871  MII != E; ++MII)
2872  if (MII->isCopyLike()) {
2873  if (applyTerminalRule(*MII))
2874  Terminals.push_back(&(*MII));
2875  else
2876  WorkList.push_back(MII);
2877  }
2878  // Append the copies evicted by the terminal rule at the end of the list.
2879  WorkList.append(Terminals.begin(), Terminals.end());
2880  }
2881  // Try coalescing the collected copies immediately, and remove the nulls.
2882  // This prevents the WorkList from getting too large since most copies are
2883  // joinable on the first attempt.
2885  CurrList(WorkList.begin() + PrevSize, WorkList.end());
2886  if (copyCoalesceWorkList(CurrList))
2887  WorkList.erase(std::remove(WorkList.begin() + PrevSize, WorkList.end(),
2888  (MachineInstr*)nullptr), WorkList.end());
2889 }
2890 
2891 void RegisterCoalescer::coalesceLocals() {
2892  copyCoalesceWorkList(LocalWorkList);
2893  for (unsigned j = 0, je = LocalWorkList.size(); j != je; ++j) {
2894  if (LocalWorkList[j])
2895  WorkList.push_back(LocalWorkList[j]);
2896  }
2897  LocalWorkList.clear();
2898 }
2899 
2900 void RegisterCoalescer::joinAllIntervals() {
2901  DEBUG(dbgs() << "********** JOINING INTERVALS ***********\n");
2902  assert(WorkList.empty() && LocalWorkList.empty() && "Old data still around.");
2903 
2904  std::vector<MBBPriorityInfo> MBBs;
2905  MBBs.reserve(MF->size());
2906  for (MachineFunction::iterator I = MF->begin(), E = MF->end();I != E;++I){
2907  MachineBasicBlock *MBB = I;
2908  MBBs.push_back(MBBPriorityInfo(MBB, Loops->getLoopDepth(MBB),
2909  JoinSplitEdges && isSplitEdge(MBB)));
2910  }
2911  array_pod_sort(MBBs.begin(), MBBs.end(), compareMBBPriority);
2912 
2913  // Coalesce intervals in MBB priority order.
2914  unsigned CurrDepth = UINT_MAX;
2915  for (unsigned i = 0, e = MBBs.size(); i != e; ++i) {
2916  // Try coalescing the collected local copies for deeper loops.
2917  if (JoinGlobalCopies && MBBs[i].Depth < CurrDepth) {
2918  coalesceLocals();
2919  CurrDepth = MBBs[i].Depth;
2920  }
2921  copyCoalesceInMBB(MBBs[i].MBB);
2922  }
2923  coalesceLocals();
2924 
2925  // Joining intervals can allow other intervals to be joined. Iteratively join
2926  // until we make no progress.
2927  while (copyCoalesceWorkList(WorkList))
2928  /* empty */ ;
2929 }
2930 
2931 void RegisterCoalescer::releaseMemory() {
2932  ErasedInstrs.clear();
2933  WorkList.clear();
2934  DeadDefs.clear();
2935  InflateRegs.clear();
2936 }
2937 
2938 bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
2939  MF = &fn;
2940  MRI = &fn.getRegInfo();
2941  TM = &fn.getTarget();
2942  const TargetSubtargetInfo &STI = fn.getSubtarget();
2943  TRI = STI.getRegisterInfo();
2944  TII = STI.getInstrInfo();
2945  LIS = &getAnalysis<LiveIntervals>();
2946  AA = &getAnalysis<AliasAnalysis>();
2947  Loops = &getAnalysis<MachineLoopInfo>();
2949  JoinGlobalCopies = STI.enableJoinGlobalCopies();
2950  else
2951  JoinGlobalCopies = (EnableGlobalCopies == cl::BOU_TRUE);
2952 
2953  // The MachineScheduler does not currently require JoinSplitEdges. This will
2954  // either be enabled unconditionally or replaced by a more general live range
2955  // splitting optimization.
2956  JoinSplitEdges = EnableJoinSplits;
2957 
2958  DEBUG(dbgs() << "********** SIMPLE REGISTER COALESCING **********\n"
2959  << "********** Function: " << MF->getName() << '\n');
2960 
2961  if (VerifyCoalescing)
2962  MF->verify(this, "Before register coalescing");
2963 
2964  RegClassInfo.runOnMachineFunction(fn);
2965 
2966  // Join (coalesce) intervals if requested.
2967  if (EnableJoining)
2968  joinAllIntervals();
2969 
2970  // After deleting a lot of copies, register classes may be less constrained.
2971  // Removing sub-register operands may allow GR32_ABCD -> GR32 and DPR_VFP2 ->
2972  // DPR inflation.
2973  array_pod_sort(InflateRegs.begin(), InflateRegs.end());
2974  InflateRegs.erase(std::unique(InflateRegs.begin(), InflateRegs.end()),
2975  InflateRegs.end());
2976  DEBUG(dbgs() << "Trying to inflate " << InflateRegs.size() << " regs.\n");
2977  for (unsigned i = 0, e = InflateRegs.size(); i != e; ++i) {
2978  unsigned Reg = InflateRegs[i];
2979  if (MRI->reg_nodbg_empty(Reg))
2980  continue;
2981  if (MRI->recomputeRegClass(Reg)) {
2982  DEBUG(dbgs() << PrintReg(Reg) << " inflated to "
2983  << TRI->getRegClassName(MRI->getRegClass(Reg)) << '\n');
2984  LiveInterval &LI = LIS->getInterval(Reg);
2985  unsigned MaxMask = MRI->getMaxLaneMaskForVReg(Reg);
2986  if (MaxMask == 0) {
2987  // If the inflated register class does not support subregisters anymore
2988  // remove the subranges.
2989  LI.clearSubRanges();
2990  } else {
2991 #ifndef NDEBUG
2992  // If subranges are still supported, then the same subregs should still
2993  // be supported.
2994  for (LiveInterval::SubRange &S : LI.subranges()) {
2995  assert ((S.LaneMask & ~MaxMask) == 0);
2996  }
2997 #endif
2998  }
2999  ++NumInflated;
3000  }
3001  }
3002 
3003  DEBUG(dump());
3004  if (VerifyCoalescing)
3005  MF->verify(this, "After register coalescing");
3006  return true;
3007 }
3008 
3009 void RegisterCoalescer::print(raw_ostream &O, const Module* m) const {
3010  LIS->print(O, m);
3011 }
bool isFullCopy() const
Definition: MachineInstr.h:781
bool isImplicit() const
unsigned succ_size() const
static bool isSplitEdge(const MachineBasicBlock *MBB)
Return true if this block should be vacated by the coalescer to eliminate branches.
void push_back(const T &Elt)
Definition: SmallVector.h:222
const MachineFunction * getParent() const
getParent - Return the MachineFunction containing this basic block.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Segments::iterator iterator
Definition: LiveInterval.h:204
const unsigned reg
Definition: LiveInterval.h:616
SlotIndex def
The index of the defining instruction.
Definition: LiveInterval.h:53
unsigned getDstReg() const
Return the register (virtual or physical) that will remain after coalescing.
STATISTIC(NumFunctions,"Total number of functions")
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
int getNumber() const
getNumber - MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a M...
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:114
INITIALIZE_PASS_BEGIN(RegisterCoalescer,"simple-register-coalescing","Simple Register Coalescing", false, false) INITIALIZE_PASS_END(RegisterCoalescer
static cl::opt< bool > VerifyCoalescing("verify-coalescing", cl::desc("Verify machine instrs before and after register coalescing"), cl::Hidden)
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:191
std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)
Remove path.
static bool isImpreciseLaneMask(unsigned LaneMask)
Returns true if the given lane mask is imprecise.
char & MachineDominatorsID
MachineDominators - This pass is a machine dominators analysis pass.
LiveInterval - This class represents the liveness of a register, or stack slot.
Definition: LiveInterval.h:588
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:138
void setIsUndef(bool Val=true)
bool isDead() const
static bool isVirtualRegister(unsigned Reg)
isVirtualRegister - Return true if the specified register number is in the virtual register namespace...
static bool isMoveInstr(const TargetRegisterInfo &tri, const MachineInstr *MI, unsigned &Src, unsigned &Dst, unsigned &SrcSub, unsigned &DstSub)
static bool isLocalCopy(MachineInstr *Copy, const LiveIntervals *LIS)
static bool definesFullReg(const MachineInstr &MI, unsigned Reg)
Returns true if MI defines the full vreg Reg, as opposed to just defining a subregister.
PrintRegUnit - Helper class for printing register units on a raw_ostream.
char & RegisterCoalescerID
RegisterCoalescer - This pass merges live ranges to eliminate copies.
bool isKill() const
Return true if the live-in value is killed by this instruction.
Definition: LiveInterval.h:108
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:264
A live range for subregisters.
Definition: LiveInterval.h:595
This represents a simple continuous liveness interval for a value.
Definition: LiveInterval.h:159
const TargetRegisterClass * getCommonSubClass(const TargetRegisterClass *A, const TargetRegisterClass *B) const
getCommonSubClass - find the largest common subclass of A and B.
void setIsDead(bool Val=true)
SlotIndex endPoint() const
Return the end point of the last live range segment to interact with the instruction, if any.
Definition: LiveInterval.h:143
void markUnused()
Mark this value as unused.
Definition: LiveInterval.h:80
VNInfo - Value Number Information.
Definition: LiveInterval.h:45
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:295
void substVirtReg(unsigned Reg, unsigned SubIdx, const TargetRegisterInfo &)
substVirtReg - Substitute the current register with the virtual subregister Reg:SubReg.
unsigned getNumValNums() const
Definition: LiveInterval.h:288
VNInfo * getVNInfoAt(SlotIndex Idx) const
getVNInfoAt - Return the VNInfo that is live at Idx, or NULL.
Definition: LiveInterval.h:392
Callback methods for LiveRangeEdit owners.
Definition: LiveRangeEdit.h:40
void initializeRegisterCoalescerPass(PassRegistry &)
bool allDefsAreDead() const
Return true if all the defs of this instruction are dead.
This class represents the liveness of a register, stack slot, etc.
Definition: LiveInterval.h:153
SubRange * createSubRangeFrom(BumpPtrAllocator &Allocator, unsigned LaneMask, const LiveRange &CopyFrom)
Like createSubRange() but the new range is filled with a copy of the liveness information in CopyFrom...
Definition: LiveInterval.h:688
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:242
void removeEmptySubRanges()
Removes all subranges without any segments (subranges without segments are not considered valid and s...
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:70
Hexagon Hardware Loops
bool isCrossClass() const
Return true if DstReg is virtual and NewRC is a smaller register class than DstReg's.
MachineBasicBlock * intervalIsInOneMBB(const LiveInterval &LI) const
intervalIsInOneMBB - If LI is confined to a single basic block, return a pointer to that block...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
static MachineOperand CreateReg(unsigned Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false)
bool isFlipped() const
Return true when getSrcReg is the register being defined by the original copy instruction.
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
Definition: SmallVector.h:406
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
iterator end()
Definition: LiveInterval.h:206
A helper class for register coalescers.
std::pair< bool, bool > readsWritesVirtualRegister(unsigned Reg, SmallVectorImpl< unsigned > *Ops=nullptr) const
Return a pair of bools (reads, writes) indicating if this instruction reads or writes Reg...
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:75
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
bool isReg() const
isReg - Tests if this is a MO_Register operand.
iterator_range< subrange_iterator > subranges()
Definition: LiveInterval.h:670
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Result of a LiveRange query.
Definition: LiveInterval.h:86
static cl::opt< bool > EnableJoinSplits("join-splitedges", cl::desc("Coalesce copies on split edges (default=subtarget)"), cl::Hidden)
Temporary flag to test critical edge unsplitting.
const TargetRegisterClass * getRegClass(unsigned Reg) const
getRegClass - Return the register class of the specified virtual register.
Reg
All possible values of the reg field in the ModR/M byte.
bool isUndef() const
iterator_range< reg_instr_nodbg_iterator > reg_nodbg_instructions(unsigned Reg) const
bool isBlock() const
isBlock - Returns true if this is a block boundary slot.
Definition: SlotIndexes.h:229
ELFYAML::ELF_STO Other
Definition: ELFYAML.cpp:591
unsigned getNumOperands() const
Access to explicit operands of the instruction.
Definition: MachineInstr.h:271
bool isUnused() const
Returns true if this value is unused.
Definition: LiveInterval.h:77
SlotIndex getNextNonNullIndex(SlotIndex Index)
Returns the next non-null index, if one exists.
Definition: SlotIndexes.h:429
defusechain_iterator - This class provides iterator support for machine operands in the function that...
virtual bool enableJoinGlobalCopies() const
True if the subtarget should enable joining global copies.
PrintReg - Helper class for printing registers on a raw_ostream.
bool isDead() const
isDead - Returns true if this is a dead def kill slot.
Definition: SlotIndexes.h:239
SlotIndexes pass.
Definition: SlotIndexes.h:334
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
iterator addSegment(Segment S)
Add the specified Segment to this range, merging segments as appropriate.
Segments segments
Definition: LiveInterval.h:195
VNInfo * MergeValueNumberInto(VNInfo *V1, VNInfo *V2)
MergeValueNumberInto - This method is called when two value numbers are found to be equivalent...
unsigned getMatchingSuperReg(unsigned Reg, unsigned SubIdx, const TargetRegisterClass *RC) const
getMatchingSuperReg - Return a super-register of the specified register Reg so its sub-register of in...
VNInfo * valueOutOrDead() const
Returns the value alive at the end of the instruction, if any.
Definition: LiveInterval.h:125
bool isCopyLike() const
Return true if the instruction behaves like a copy.
Definition: MachineInstr.h:790
virtual const TargetRegisterClass * getMatchingSuperRegClass(const TargetRegisterClass *A, const TargetRegisterClass *B, unsigned Idx) const
getMatchingSuperRegClass - Return a subclass of the specified register class A so that each register ...
AnalysisUsage & addPreservedID(const void *ID)
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:134
bool setRegisters(const MachineInstr *)
Set registers to match the copy instruction MI.
SlotIndex getPrevSlot() const
Returns the previous slot in the index list.
Definition: SlotIndexes.h:292
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:120
format_object< Ts...> format(const char *Fmt, const Ts &...Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:111
TargetInstrInfo - Interface to description of machine instruction set.
bool isDebugValue() const
Definition: MachineInstr.h:748
bool isImplicitDef() const
Definition: MachineInstr.h:759
const TargetRegisterClass * getCommonSuperRegClass(const TargetRegisterClass *RCA, unsigned SubA, const TargetRegisterClass *RCB, unsigned SubB, unsigned &PreA, unsigned &PreB) const
getCommonSuperRegClass - Find a common super-register class if it exists.
bundle_iterator< MachineInstr, instr_iterator > iterator
void removeValNo(VNInfo *ValNo)
removeValNo - Remove all the segments defined by the specified value#.
const TargetRegisterClass * getNewRC() const
Return the register class of the coalesced register.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
void array_pod_sort(IteratorTy Start, IteratorTy End)
array_pod_sort - This sorts an array with the specified start and end extent.
Definition: STLExtras.h:287
LiveQueryResult Query(SlotIndex Idx) const
Query Liveness at Idx.
Definition: LiveInterval.h:493
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:221
static int compareMBBPriority(const MBBPriorityInfo *LHS, const MBBPriorityInfo *RHS)
C-style comparator that sorts first based on the loop depth of the basic block (the unsigned)...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Allocate memory in an ever growing pool, as if by bump-pointer.
Definition: Allocator.h:135
static void addSegmentsWithValNo(LiveRange &Dst, VNInfo *DstValNo, const LiveRange &Src, const VNInfo *SrcValNo)
Copy segements with value number SrcValNo from liverange Src to live range and use value number DstV...
int findRegisterDefOperandIdx(unsigned Reg, bool isDead=false, bool Overlap=false, const TargetRegisterInfo *TRI=nullptr) const
Returns the operand index that is a def of the specified register or -1 if it is not found...
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:273
unsigned getSubReg(unsigned Reg, unsigned Idx) const
Returns the physical register number of sub-register "Index" for physical register RegNo...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:264
bool isRegTiedToUseOperand(unsigned DefOpIdx, unsigned *UseOpIdx=nullptr) const
Given the index of a register def operand, check if the register def is tied to a source operand...
Definition: MachineInstr.h:995
bool isCopy() const
Definition: MachineInstr.h:778
Represent the analysis usage information of a pass.
void substPhysReg(unsigned Reg, const TargetRegisterInfo &)
substPhysReg - Substitute the current register with the physical register Reg, taking any existing Su...
VNInfo * valueDefined() const
Return the value defined by this instruction, if any.
Definition: LiveInterval.h:131
static bool isTerminalReg(unsigned DstReg, const MachineInstr &Copy, const MachineRegisterInfo *MRI)
Check if DstReg is a terminal node.
void verify() const
Walk the range and assert if any invariants fail to hold.
#define INITIALIZE_AG_DEPENDENCY(depName)
Definition: PassSupport.h:72
unsigned getSubReg() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool liveAt(SlotIndex index) const
Definition: LiveInterval.h:372
bool isPHIDef() const
Returns true if this value is defined by a PHI instruction (or was, PHI instructions may have been el...
Definition: LiveInterval.h:74
MachineInstr * getInstructionFromIndex(SlotIndex index) const
Returns the instruction for the given index, or null if the given index has no instruction associated...
Definition: SlotIndexes.h:423
static cl::opt< cl::boolOrDefault > EnableGlobalCopies("join-globalcopies", cl::desc("Coalesce copies that span blocks (default=subtarget)"), cl::init(cl::BOU_UNSET), cl::Hidden)
Temporary flag to test global copy optimization.
void setIsKill(bool Val=true)
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
static cl::opt< bool > EnableJoining("join-liveintervals", cl::desc("Coalesce copies (default=true)"), cl::init(true))
VNInfo * valueOut() const
Return the value leaving the instruction, if any.
Definition: LiveInterval.h:119
bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const
Return true if it is safe to move this instruction.
bool readsRegister(unsigned Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr reads the specified register.
Definition: MachineInstr.h:836
int findRegisterUseOperandIdx(unsigned Reg, bool isKill=false, const TargetRegisterInfo *TRI=nullptr) const
Returns the operand index that is a use of the specific register or -1 if it is not found...
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:299
bool isPhys() const
Return true if DstReg is a physical register.
void substituteRegister(unsigned FromReg, unsigned ToReg, unsigned SubIdx, const TargetRegisterInfo &RegInfo)
Replace all occurrences of FromReg with ToReg:SubIdx, properly composing subreg indices where necessa...
void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false...
Definition: SmallPtrSet.h:271
static bool isSameInstr(SlotIndex A, SlotIndex B)
isSameInstr - Return true if A and B refer to the same instruction.
Definition: SlotIndexes.h:206
simple register Simple Register false
simple register coalescing
ConnectedVNInfoEqClasses - Helper class that can divide VNInfos in a LiveInterval into equivalence cl...
Definition: LiveInterval.h:841
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
MachineBasicBlock * getMBBFromIndex(SlotIndex index) const
Returns the basic block which the given index falls in.
Definition: SlotIndexes.h:506
Promote Memory to Register
Definition: Mem2Reg.cpp:58
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:263
LiveInterval & getInterval(unsigned Reg)
bool isCoalescable(const MachineInstr *) const
Return true if MI is a copy instruction that will become an identity copy after coalescing.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
void clearSubRanges()
Removes all subregister liveness information.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:576
SlotIndex getMBBEndIdx(unsigned Num) const
Returns the last index in the given basic block number.
Definition: SlotIndexes.h:496
static bool clobbersPhysReg(const uint32_t *RegMask, unsigned PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
StringRef getName() const
getName - Return the name of the corresponding LLVM basic block, or "(null)".
unsigned getDstIdx() const
Return the subregister index that DstReg will be coalesced into, or 0.
VNInfo * getValNumInfo(unsigned ValNo)
getValNumInfo - Returns pointer to the specified val#.
Definition: LiveInterval.h:292
bool containsOneValue() const
Definition: LiveInterval.h:286
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
TargetSubtargetInfo - Generic base class for all target subtargets.
#define Success
unsigned getSrcIdx() const
Return the subregister index that SrcReg will be coalesced into, or 0.
Representation of each machine instruction.
Definition: MachineInstr.h:51
static bool isPhysicalRegister(unsigned Reg)
isPhysicalRegister - Return true if the specified register number is in the physical register namespa...
static cl::opt< bool > UseTerminalRule("terminal-rule", cl::desc("Apply the terminal rule"), cl::init(false), cl::Hidden)
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
VNInfo * createDeadDef(SlotIndex Def, VNInfo::Allocator &VNInfoAllocator)
createDeadDef - Make sure the range has a value defined at Def.
SubRange * createSubRange(BumpPtrAllocator &Allocator, unsigned LaneMask)
Creates a new empty subregister live range.
Definition: LiveInterval.h:680
void setReg(unsigned Reg)
Change the register this operand corresponds to.
#define I(x, y, z)
Definition: MD5.cpp:54
void setSubReg(unsigned subReg)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
void size_t size
bool flip()
Swap SrcReg and DstReg.
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def...
Definition: SlotIndexes.h:257
iterator begin()
Definition: LiveInterval.h:205
unsigned getReg() const
getReg - Returns the register number.
VNInfo * getNextValue(SlotIndex def, VNInfo::Allocator &VNInfoAllocator)
getNextValue - Create a new value number and return it.
Definition: LiveInterval.h:306
bool isCommutable(QueryType Type=IgnoreBundle) const
Return true if this may be a 2- or 3-address instruction (of the form "X = op Y, Z, ..."), which produces the same result if Y and Z are exchanged.
Definition: MachineInstr.h:607
bool isPartial() const
Return true if the original copy instruction did not copy the full register, but was a subreg operati...
aarch64 promote const
void eliminateDeadDefs(SmallVectorImpl< MachineInstr * > &Dead, ArrayRef< unsigned > RegsBeingSpilled=None)
eliminateDeadDefs - Try to delete machine instructions that are now dead (allDefsAreDead returns true...
virtual const TargetInstrInfo * getInstrInfo() const
simple register Simple Register Coalescing
unsigned getSrcReg() const
Return the virtual register that will be coalesced away.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:185
BasicBlockListType::iterator iterator
This class implements an extremely fast bulk output stream that can only output to a stream...
Definition: raw_ostream.h:38
#define DEBUG(X)
Definition: Debug.h:92
Primary interface to the complete machine description for the target machine.
bool isRegTiedToDefOperand(unsigned UseOpIdx, unsigned *DefOpIdx=nullptr) const
Return true if the use operand of the specified index is tied to a def operand.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
unsigned composeSubRegIndices(unsigned a, unsigned b) const
composeSubRegIndices - Return the subregister index you get from composing two subregister indices...
void join(LiveRange &Other, const int *ValNoAssignments, const int *RHSValNoAssignments, SmallVectorImpl< VNInfo * > &NewVNInfo)
join - Join two live ranges (this, and other) together.
iterator FindSegmentContaining(SlotIndex Idx)
Return an iterator to the segment that contains the specified index, or end() if there is none...
Definition: LiveInterval.h:407
bool readsReg() const
readsReg - Returns true if this operand reads the previous value of its register. ...
VNInfo * valueIn() const
Return the value that is live-in to the instruction.
Definition: LiveInterval.h:101
bool hasSubRanges() const
Returns true if subregister liveness information is available.
Definition: LiveInterval.h:696
SlotIndex - An opaque wrapper around machine indexes.
Definition: SlotIndexes.h:92
reg_begin/reg_end - Provide iteration support to walk over all definitions and uses of a register wit...
unsigned pred_size() const
bool contains(unsigned Reg) const
contains - Return true if the specified register is included in this register class.