LLVM  3.7.0
AArch64A57FPLoadBalancing.cpp
Go to the documentation of this file.
1 //===-- AArch64A57FPLoadBalancing.cpp - Balance FP ops statically on A57---===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 // For best-case performance on Cortex-A57, we should try to use a balanced
10 // mix of odd and even D-registers when performing a critical sequence of
11 // independent, non-quadword FP/ASIMD floating-point multiply or
12 // multiply-accumulate operations.
13 //
14 // This pass attempts to detect situations where the register allocation may
15 // adversely affect this load balancing and to change the registers used so as
16 // to better utilize the CPU.
17 //
18 // Ideally we'd just take each multiply or multiply-accumulate in turn and
19 // allocate it alternating even or odd registers. However, multiply-accumulates
20 // are most efficiently performed in the same functional unit as their
21 // accumulation operand. Therefore this pass tries to find maximal sequences
22 // ("Chains") of multiply-accumulates linked via their accumulation operand,
23 // and assign them all the same "color" (oddness/evenness).
24 //
25 // This optimization affects S-register and D-register floating point
26 // multiplies and FMADD/FMAs, as well as vector (floating point only) muls and
27 // FMADD/FMA. Q register instructions (and 128-bit vector instructions) are
28 // not affected.
29 //===----------------------------------------------------------------------===//
30 
31 #include "AArch64.h"
32 #include "AArch64InstrInfo.h"
33 #include "AArch64Subtarget.h"
34 #include "llvm/ADT/BitVector.h"
44 #include "llvm/Support/Debug.h"
46 #include <list>
47 using namespace llvm;
48 
49 #define DEBUG_TYPE "aarch64-a57-fp-load-balancing"
50 
51 // Enforce the algorithm to use the scavenged register even when the original
52 // destination register is the correct color. Used for testing.
53 static cl::opt<bool>
54 TransformAll("aarch64-a57-fp-load-balancing-force-all",
55  cl::desc("Always modify dest registers regardless of color"),
56  cl::init(false), cl::Hidden);
57 
58 // Never use the balance information obtained from chains - return a specific
59 // color always. Used for testing.
60 static cl::opt<unsigned>
61 OverrideBalance("aarch64-a57-fp-load-balancing-override",
62  cl::desc("Ignore balance information, always return "
63  "(1: Even, 2: Odd)."),
64  cl::init(0), cl::Hidden);
65 
66 //===----------------------------------------------------------------------===//
67 // Helper functions
68 
69 // Is the instruction a type of multiply on 64-bit (or 32-bit) FPRs?
70 static bool isMul(MachineInstr *MI) {
71  switch (MI->getOpcode()) {
72  case AArch64::FMULSrr:
73  case AArch64::FNMULSrr:
74  case AArch64::FMULDrr:
75  case AArch64::FNMULDrr:
76  return true;
77  default:
78  return false;
79  }
80 }
81 
82 // Is the instruction a type of FP multiply-accumulate on 64-bit (or 32-bit) FPRs?
83 static bool isMla(MachineInstr *MI) {
84  switch (MI->getOpcode()) {
85  case AArch64::FMSUBSrrr:
86  case AArch64::FMADDSrrr:
87  case AArch64::FNMSUBSrrr:
88  case AArch64::FNMADDSrrr:
89  case AArch64::FMSUBDrrr:
90  case AArch64::FMADDDrrr:
91  case AArch64::FNMSUBDrrr:
92  case AArch64::FNMADDDrrr:
93  return true;
94  default:
95  return false;
96  }
97 }
98 
99 namespace llvm {
101 }
102 
103 //===----------------------------------------------------------------------===//
104 
105 namespace {
106 /// A "color", which is either even or odd. Yes, these aren't really colors
107 /// but the algorithm is conceptually doing two-color graph coloring.
108 enum class Color { Even, Odd };
109 #ifndef NDEBUG
110 static const char *ColorNames[2] = { "Even", "Odd" };
111 #endif
112 
113 class Chain;
114 
115 class AArch64A57FPLoadBalancing : public MachineFunctionPass {
116  MachineRegisterInfo *MRI;
117  const TargetRegisterInfo *TRI;
118  RegisterClassInfo RCI;
119 
120 public:
121  static char ID;
122  explicit AArch64A57FPLoadBalancing() : MachineFunctionPass(ID) {
124  }
125 
126  bool runOnMachineFunction(MachineFunction &F) override;
127 
128  const char *getPassName() const override {
129  return "A57 FP Anti-dependency breaker";
130  }
131 
132  void getAnalysisUsage(AnalysisUsage &AU) const override {
133  AU.setPreservesCFG();
135  }
136 
137 private:
138  bool runOnBasicBlock(MachineBasicBlock &MBB);
139  bool colorChainSet(std::vector<Chain*> GV, MachineBasicBlock &MBB,
140  int &Balance);
141  bool colorChain(Chain *G, Color C, MachineBasicBlock &MBB);
142  int scavengeRegister(Chain *G, Color C, MachineBasicBlock &MBB);
143  void scanInstruction(MachineInstr *MI, unsigned Idx,
144  std::map<unsigned, Chain*> &Active,
145  std::vector<std::unique_ptr<Chain>> &AllChains);
146  void maybeKillChain(MachineOperand &MO, unsigned Idx,
147  std::map<unsigned, Chain*> &RegChains);
148  Color getColor(unsigned Register);
149  Chain *getAndEraseNext(Color PreferredColor, std::vector<Chain*> &L);
150 };
151 }
152 
154 
155 INITIALIZE_PASS_BEGIN(AArch64A57FPLoadBalancing, DEBUG_TYPE,
156  "AArch64 A57 FP Load-Balancing", false, false)
157 INITIALIZE_PASS_END(AArch64A57FPLoadBalancing, DEBUG_TYPE,
158  "AArch64 A57 FP Load-Balancing", false, false)
159 
160 namespace {
161 /// A Chain is a sequence of instructions that are linked together by
162 /// an accumulation operand. For example:
163 ///
164 /// fmul d0<def>, ?
165 /// fmla d1<def>, ?, ?, d0<kill>
166 /// fmla d2<def>, ?, ?, d1<kill>
167 ///
168 /// There may be other instructions interleaved in the sequence that
169 /// do not belong to the chain. These other instructions must not use
170 /// the "chain" register at any point.
171 ///
172 /// We currently only support chains where the "chain" operand is killed
173 /// at each link in the chain for simplicity.
174 /// A chain has three important instructions - Start, Last and Kill.
175 /// * The start instruction is the first instruction in the chain.
176 /// * Last is the final instruction in the chain.
177 /// * Kill may or may not be defined. If defined, Kill is the instruction
178 /// where the outgoing value of the Last instruction is killed.
179 /// This information is important as if we know the outgoing value is
180 /// killed with no intervening uses, we can safely change its register.
181 ///
182 /// Without a kill instruction, we must assume the outgoing value escapes
183 /// beyond our model and either must not change its register or must
184 /// create a fixup FMOV to keep the old register value consistent.
185 ///
186 class Chain {
187 public:
188  /// The important (marker) instructions.
189  MachineInstr *StartInst, *LastInst, *KillInst;
190  /// The index, from the start of the basic block, that each marker
191  /// appears. These are stored so we can do quick interval tests.
192  unsigned StartInstIdx, LastInstIdx, KillInstIdx;
193  /// All instructions in the chain.
194  std::set<MachineInstr*> Insts;
195  /// True if KillInst cannot be modified. If this is true,
196  /// we cannot change LastInst's outgoing register.
197  /// This will be true for tied values and regmasks.
199  /// The "color" of LastInst. This will be the preferred chain color,
200  /// as changing intermediate nodes is easy but changing the last
201  /// instruction can be more tricky.
203 
204  Chain(MachineInstr *MI, unsigned Idx, Color C)
205  : StartInst(MI), LastInst(MI), KillInst(nullptr),
206  StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0),
207  LastColor(C) {
208  Insts.insert(MI);
209  }
210 
211  /// Add a new instruction into the chain. The instruction's dest operand
212  /// has the given color.
213  void add(MachineInstr *MI, unsigned Idx, Color C) {
214  LastInst = MI;
215  LastInstIdx = Idx;
216  LastColor = C;
217  assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) &&
218  "Chain: broken invariant. A Chain can only be killed after its last "
219  "def");
220 
221  Insts.insert(MI);
222  }
223 
224  /// Return true if MI is a member of the chain.
225  bool contains(MachineInstr *MI) { return Insts.count(MI) > 0; }
226 
227  /// Return the number of instructions in the chain.
228  unsigned size() const {
229  return Insts.size();
230  }
231 
232  /// Inform the chain that its last active register (the dest register of
233  /// LastInst) is killed by MI with no intervening uses or defs.
234  void setKill(MachineInstr *MI, unsigned Idx, bool Immutable) {
235  KillInst = MI;
236  KillInstIdx = Idx;
237  KillIsImmutable = Immutable;
238  assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) &&
239  "Chain: broken invariant. A Chain can only be killed after its last "
240  "def");
241  }
242 
243  /// Return the first instruction in the chain.
244  MachineInstr *getStart() const { return StartInst; }
245  /// Return the last instruction in the chain.
246  MachineInstr *getLast() const { return LastInst; }
247  /// Return the "kill" instruction (as set with setKill()) or NULL.
248  MachineInstr *getKill() const { return KillInst; }
249  /// Return an instruction that can be used as an iterator for the end
250  /// of the chain. This is the maximum of KillInst (if set) and LastInst.
252  return ++MachineBasicBlock::iterator(KillInst ? KillInst : LastInst);
253  }
254 
255  /// Can the Kill instruction (assuming one exists) be modified?
256  bool isKillImmutable() const { return KillIsImmutable; }
257 
258  /// Return the preferred color of this chain.
260  if (OverrideBalance != 0)
261  return OverrideBalance == 1 ? Color::Even : Color::Odd;
262  return LastColor;
263  }
264 
265  /// Return true if this chain (StartInst..KillInst) overlaps with Other.
266  bool rangeOverlapsWith(const Chain &Other) const {
267  unsigned End = KillInst ? KillInstIdx : LastInstIdx;
268  unsigned OtherEnd = Other.KillInst ?
269  Other.KillInstIdx : Other.LastInstIdx;
270 
271  return StartInstIdx <= OtherEnd && Other.StartInstIdx <= End;
272  }
273 
274  /// Return true if this chain starts before Other.
275  bool startsBefore(const Chain *Other) const {
276  return StartInstIdx < Other->StartInstIdx;
277  }
278 
279  /// Return true if the group will require a fixup MOV at the end.
280  bool requiresFixup() const {
281  return (getKill() && isKillImmutable()) || !getKill();
282  }
283 
284  /// Return a simple string representation of the chain.
285  std::string str() const {
286  std::string S;
287  raw_string_ostream OS(S);
288 
289  OS << "{";
290  StartInst->print(OS, /* SkipOpers= */true);
291  OS << " -> ";
292  LastInst->print(OS, /* SkipOpers= */true);
293  if (KillInst) {
294  OS << " (kill @ ";
295  KillInst->print(OS, /* SkipOpers= */true);
296  OS << ")";
297  }
298  OS << "}";
299 
300  return OS.str();
301  }
302 
303 };
304 
305 } // end anonymous namespace
306 
307 //===----------------------------------------------------------------------===//
308 
309 bool AArch64A57FPLoadBalancing::runOnMachineFunction(MachineFunction &F) {
310  // Don't do anything if this isn't an A53 or A57.
312  F.getSubtarget<AArch64Subtarget>().isCortexA57()))
313  return false;
314 
315  bool Changed = false;
316  DEBUG(dbgs() << "***** AArch64A57FPLoadBalancing *****\n");
317 
318  MRI = &F.getRegInfo();
319  TRI = F.getRegInfo().getTargetRegisterInfo();
320  RCI.runOnMachineFunction(F);
321 
322  for (auto &MBB : F) {
323  Changed |= runOnBasicBlock(MBB);
324  }
325 
326  return Changed;
327 }
328 
329 bool AArch64A57FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) {
330  bool Changed = false;
331  DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n");
332 
333  // First, scan the basic block producing a set of chains.
334 
335  // The currently "active" chains - chains that can be added to and haven't
336  // been killed yet. This is keyed by register - all chains can only have one
337  // "link" register between each inst in the chain.
338  std::map<unsigned, Chain*> ActiveChains;
339  std::vector<std::unique_ptr<Chain>> AllChains;
340  unsigned Idx = 0;
341  for (auto &MI : MBB)
342  scanInstruction(&MI, Idx++, ActiveChains, AllChains);
343 
344  DEBUG(dbgs() << "Scan complete, "<< AllChains.size() << " chains created.\n");
345 
346  // Group the chains into disjoint sets based on their liveness range. This is
347  // a poor-man's version of graph coloring. Ideally we'd create an interference
348  // graph and perform full-on graph coloring on that, but;
349  // (a) That's rather heavyweight for only two colors.
350  // (b) We expect multiple disjoint interference regions - in practice the live
351  // range of chains is quite small and they are clustered between loads
352  // and stores.
354  for (auto &I : AllChains)
355  EC.insert(I.get());
356 
357  for (auto &I : AllChains)
358  for (auto &J : AllChains)
359  if (I != J && I->rangeOverlapsWith(*J))
360  EC.unionSets(I.get(), J.get());
361  DEBUG(dbgs() << "Created " << EC.getNumClasses() << " disjoint sets.\n");
362 
363  // Now we assume that every member of an equivalence class interferes
364  // with every other member of that class, and with no members of other classes.
365 
366  // Convert the EquivalenceClasses to a simpler set of sets.
367  std::vector<std::vector<Chain*> > V;
368  for (auto I = EC.begin(), E = EC.end(); I != E; ++I) {
369  std::vector<Chain*> Cs(EC.member_begin(I), EC.member_end());
370  if (Cs.empty()) continue;
371  V.push_back(std::move(Cs));
372  }
373 
374  // Now we have a set of sets, order them by start address so
375  // we can iterate over them sequentially.
376  std::sort(V.begin(), V.end(),
377  [](const std::vector<Chain*> &A,
378  const std::vector<Chain*> &B) {
379  return A.front()->startsBefore(B.front());
380  });
381 
382  // As we only have two colors, we can track the global (BB-level) balance of
383  // odds versus evens. We aim to keep this near zero to keep both execution
384  // units fed.
385  // Positive means we're even-heavy, negative we're odd-heavy.
386  //
387  // FIXME: If chains have interdependencies, for example:
388  // mul r0, r1, r2
389  // mul r3, r0, r1
390  // We do not model this and may color each one differently, assuming we'll
391  // get ILP when we obviously can't. This hasn't been seen to be a problem
392  // in practice so far, so we simplify the algorithm by ignoring it.
393  int Parity = 0;
394 
395  for (auto &I : V)
396  Changed |= colorChainSet(std::move(I), MBB, Parity);
397 
398  return Changed;
399 }
400 
401 Chain *AArch64A57FPLoadBalancing::getAndEraseNext(Color PreferredColor,
402  std::vector<Chain*> &L) {
403  if (L.empty())
404  return nullptr;
405 
406  // We try and get the best candidate from L to color next, given that our
407  // preferred color is "PreferredColor". L is ordered from larger to smaller
408  // chains. It is beneficial to color the large chains before the small chains,
409  // but if we can't find a chain of the maximum length with the preferred color,
410  // we fuzz the size and look for slightly smaller chains before giving up and
411  // returning a chain that must be recolored.
412 
413  // FIXME: Does this need to be configurable?
414  const unsigned SizeFuzz = 1;
415  unsigned MinSize = L.front()->size() - SizeFuzz;
416  for (auto I = L.begin(), E = L.end(); I != E; ++I) {
417  if ((*I)->size() <= MinSize) {
418  // We've gone past the size limit. Return the previous item.
419  Chain *Ch = *--I;
420  L.erase(I);
421  return Ch;
422  }
423 
424  if ((*I)->getPreferredColor() == PreferredColor) {
425  Chain *Ch = *I;
426  L.erase(I);
427  return Ch;
428  }
429  }
430 
431  // Bailout case - just return the first item.
432  Chain *Ch = L.front();
433  L.erase(L.begin());
434  return Ch;
435 }
436 
437 bool AArch64A57FPLoadBalancing::colorChainSet(std::vector<Chain*> GV,
438  MachineBasicBlock &MBB,
439  int &Parity) {
440  bool Changed = false;
441  DEBUG(dbgs() << "colorChainSet(): #sets=" << GV.size() << "\n");
442 
443  // Sort by descending size order so that we allocate the most important
444  // sets first.
445  // Tie-break equivalent sizes by sorting chains requiring fixups before
446  // those without fixups. The logic here is that we should look at the
447  // chains that we cannot change before we look at those we can,
448  // so the parity counter is updated and we know what color we should
449  // change them to!
450  // Final tie-break with instruction order so pass output is stable (i.e. not
451  // dependent on malloc'd pointer values).
452  std::sort(GV.begin(), GV.end(), [](const Chain *G1, const Chain *G2) {
453  if (G1->size() != G2->size())
454  return G1->size() > G2->size();
455  if (G1->requiresFixup() != G2->requiresFixup())
456  return G1->requiresFixup() > G2->requiresFixup();
457  // Make sure startsBefore() produces a stable final order.
458  assert((G1 == G2 || (G1->startsBefore(G2) ^ G2->startsBefore(G1))) &&
459  "Starts before not total order!");
460  return G1->startsBefore(G2);
461  });
462 
463  Color PreferredColor = Parity < 0 ? Color::Even : Color::Odd;
464  while (Chain *G = getAndEraseNext(PreferredColor, GV)) {
465  // Start off by assuming we'll color to our own preferred color.
466  Color C = PreferredColor;
467  if (Parity == 0)
468  // But if we really don't care, use the chain's preferred color.
469  C = G->getPreferredColor();
470 
471  DEBUG(dbgs() << " - Parity=" << Parity << ", Color="
472  << ColorNames[(int)C] << "\n");
473 
474  // If we'll need a fixup FMOV, don't bother. Testing has shown that this
475  // happens infrequently and when it does it has at least a 50% chance of
476  // slowing code down instead of speeding it up.
477  if (G->requiresFixup() && C != G->getPreferredColor()) {
478  C = G->getPreferredColor();
479  DEBUG(dbgs() << " - " << G->str() << " - not worthwhile changing; "
480  "color remains " << ColorNames[(int)C] << "\n");
481  }
482 
483  Changed |= colorChain(G, C, MBB);
484 
485  Parity += (C == Color::Even) ? G->size() : -G->size();
486  PreferredColor = Parity < 0 ? Color::Even : Color::Odd;
487  }
488 
489  return Changed;
490 }
491 
492 int AArch64A57FPLoadBalancing::scavengeRegister(Chain *G, Color C,
493  MachineBasicBlock &MBB) {
494  RegScavenger RS;
495  RS.enterBasicBlock(&MBB);
496  RS.forward(MachineBasicBlock::iterator(G->getStart()));
497 
498  // Can we find an appropriate register that is available throughout the life
499  // of the chain?
500  unsigned RegClassID = G->getStart()->getDesc().OpInfo[0].RegClass;
501  BitVector AvailableRegs = RS.getRegsAvailable(TRI->getRegClass(RegClassID));
502  for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd();
503  I != E; ++I) {
504  RS.forward(I);
505  AvailableRegs &= RS.getRegsAvailable(TRI->getRegClass(RegClassID));
506 
507  // Remove any registers clobbered by a regmask or any def register that is
508  // immediately dead.
509  for (auto J : I->operands()) {
510  if (J.isRegMask())
511  AvailableRegs.clearBitsNotInMask(J.getRegMask());
512 
513  if (J.isReg() && J.isDef()) {
514  MCRegAliasIterator AI(J.getReg(), TRI, /*IncludeSelf=*/true);
515  if (J.isDead())
516  for (; AI.isValid(); ++AI)
517  AvailableRegs.reset(*AI);
518 #ifndef NDEBUG
519  else
520  for (; AI.isValid(); ++AI)
521  assert(!AvailableRegs[*AI] &&
522  "Non-dead def should have been removed by now!");
523 #endif
524  }
525  }
526  }
527 
528  // Make sure we allocate in-order, to get the cheapest registers first.
529  auto Ord = RCI.getOrder(TRI->getRegClass(RegClassID));
530  for (auto Reg : Ord) {
531  if (!AvailableRegs[Reg])
532  continue;
533  if ((C == Color::Even && (Reg % 2) == 0) ||
534  (C == Color::Odd && (Reg % 2) == 1))
535  return Reg;
536  }
537 
538  return -1;
539 }
540 
541 bool AArch64A57FPLoadBalancing::colorChain(Chain *G, Color C,
542  MachineBasicBlock &MBB) {
543  bool Changed = false;
544  DEBUG(dbgs() << " - colorChain(" << G->str() << ", "
545  << ColorNames[(int)C] << ")\n");
546 
547  // Try and obtain a free register of the right class. Without a register
548  // to play with we cannot continue.
549  int Reg = scavengeRegister(G, C, MBB);
550  if (Reg == -1) {
551  DEBUG(dbgs() << "Scavenging (thus coloring) failed!\n");
552  return false;
553  }
554  DEBUG(dbgs() << " - Scavenged register: " << TRI->getName(Reg) << "\n");
555 
556  std::map<unsigned, unsigned> Substs;
557  for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd();
558  I != E; ++I) {
559  if (!G->contains(I) &&
560  (&*I != G->getKill() || G->isKillImmutable()))
561  continue;
562 
563  // I is a member of G, or I is a mutable instruction that kills G.
564 
565  std::vector<unsigned> ToErase;
566  for (auto &U : I->operands()) {
567  if (U.isReg() && U.isUse() && Substs.find(U.getReg()) != Substs.end()) {
568  unsigned OrigReg = U.getReg();
569  U.setReg(Substs[OrigReg]);
570  if (U.isKill())
571  // Don't erase straight away, because there may be other operands
572  // that also reference this substitution!
573  ToErase.push_back(OrigReg);
574  } else if (U.isRegMask()) {
575  for (auto J : Substs) {
576  if (U.clobbersPhysReg(J.first))
577  ToErase.push_back(J.first);
578  }
579  }
580  }
581  // Now it's safe to remove the substs identified earlier.
582  for (auto J : ToErase)
583  Substs.erase(J);
584 
585  // Only change the def if this isn't the last instruction.
586  if (&*I != G->getKill()) {
587  MachineOperand &MO = I->getOperand(0);
588 
589  bool Change = TransformAll || getColor(MO.getReg()) != C;
590  if (G->requiresFixup() && &*I == G->getLast())
591  Change = false;
592 
593  if (Change) {
594  Substs[MO.getReg()] = Reg;
595  MO.setReg(Reg);
596  MRI->setPhysRegUsed(Reg);
597 
598  Changed = true;
599  }
600  }
601  }
602  assert(Substs.size() == 0 && "No substitutions should be left active!");
603 
604  if (G->getKill()) {
605  DEBUG(dbgs() << " - Kill instruction seen.\n");
606  } else {
607  // We didn't have a kill instruction, but we didn't seem to need to change
608  // the destination register anyway.
609  DEBUG(dbgs() << " - Destination register not changed.\n");
610  }
611  return Changed;
612 }
613 
614 void AArch64A57FPLoadBalancing::scanInstruction(
615  MachineInstr *MI, unsigned Idx, std::map<unsigned, Chain *> &ActiveChains,
616  std::vector<std::unique_ptr<Chain>> &AllChains) {
617  // Inspect "MI", updating ActiveChains and AllChains.
618 
619  if (isMul(MI)) {
620 
621  for (auto &I : MI->uses())
622  maybeKillChain(I, Idx, ActiveChains);
623  for (auto &I : MI->defs())
624  maybeKillChain(I, Idx, ActiveChains);
625 
626  // Create a new chain. Multiplies don't require forwarding so can go on any
627  // unit.
628  unsigned DestReg = MI->getOperand(0).getReg();
629 
630  DEBUG(dbgs() << "New chain started for register "
631  << TRI->getName(DestReg) << " at " << *MI);
632 
633  auto G = llvm::make_unique<Chain>(MI, Idx, getColor(DestReg));
634  ActiveChains[DestReg] = G.get();
635  AllChains.push_back(std::move(G));
636 
637  } else if (isMla(MI)) {
638 
639  // It is beneficial to keep MLAs on the same functional unit as their
640  // accumulator operand.
641  unsigned DestReg = MI->getOperand(0).getReg();
642  unsigned AccumReg = MI->getOperand(3).getReg();
643 
644  maybeKillChain(MI->getOperand(1), Idx, ActiveChains);
645  maybeKillChain(MI->getOperand(2), Idx, ActiveChains);
646  if (DestReg != AccumReg)
647  maybeKillChain(MI->getOperand(0), Idx, ActiveChains);
648 
649  if (ActiveChains.find(AccumReg) != ActiveChains.end()) {
650  DEBUG(dbgs() << "Chain found for accumulator register "
651  << TRI->getName(AccumReg) << " in MI " << *MI);
652 
653  // For simplicity we only chain together sequences of MULs/MLAs where the
654  // accumulator register is killed on each instruction. This means we don't
655  // need to track other uses of the registers we want to rewrite.
656  //
657  // FIXME: We could extend to handle the non-kill cases for more coverage.
658  if (MI->getOperand(3).isKill()) {
659  // Add to chain.
660  DEBUG(dbgs() << "Instruction was successfully added to chain.\n");
661  ActiveChains[AccumReg]->add(MI, Idx, getColor(DestReg));
662  // Handle cases where the destination is not the same as the accumulator.
663  if (DestReg != AccumReg) {
664  ActiveChains[DestReg] = ActiveChains[AccumReg];
665  ActiveChains.erase(AccumReg);
666  }
667  return;
668  }
669 
670  DEBUG(dbgs() << "Cannot add to chain because accumulator operand wasn't "
671  << "marked <kill>!\n");
672  maybeKillChain(MI->getOperand(3), Idx, ActiveChains);
673  }
674 
675  DEBUG(dbgs() << "Creating new chain for dest register "
676  << TRI->getName(DestReg) << "\n");
677  auto G = llvm::make_unique<Chain>(MI, Idx, getColor(DestReg));
678  ActiveChains[DestReg] = G.get();
679  AllChains.push_back(std::move(G));
680 
681  } else {
682 
683  // Non-MUL or MLA instruction. Invalidate any chain in the uses or defs
684  // lists.
685  for (auto &I : MI->uses())
686  maybeKillChain(I, Idx, ActiveChains);
687  for (auto &I : MI->defs())
688  maybeKillChain(I, Idx, ActiveChains);
689 
690  }
691 }
692 
693 void AArch64A57FPLoadBalancing::
694 maybeKillChain(MachineOperand &MO, unsigned Idx,
695  std::map<unsigned, Chain*> &ActiveChains) {
696  // Given an operand and the set of active chains (keyed by register),
697  // determine if a chain should be ended and remove from ActiveChains.
698  MachineInstr *MI = MO.getParent();
699 
700  if (MO.isReg()) {
701 
702  // If this is a KILL of a current chain, record it.
703  if (MO.isKill() && ActiveChains.find(MO.getReg()) != ActiveChains.end()) {
704  DEBUG(dbgs() << "Kill seen for chain " << TRI->getName(MO.getReg())
705  << "\n");
706  ActiveChains[MO.getReg()]->setKill(MI, Idx, /*Immutable=*/MO.isTied());
707  }
708  ActiveChains.erase(MO.getReg());
709 
710  } else if (MO.isRegMask()) {
711 
712  for (auto I = ActiveChains.begin(), E = ActiveChains.end();
713  I != E;) {
714  if (MO.clobbersPhysReg(I->first)) {
715  DEBUG(dbgs() << "Kill (regmask) seen for chain "
716  << TRI->getName(I->first) << "\n");
717  I->second->setKill(MI, Idx, /*Immutable=*/true);
718  ActiveChains.erase(I++);
719  } else
720  ++I;
721  }
722 
723  }
724 }
725 
726 Color AArch64A57FPLoadBalancing::getColor(unsigned Reg) {
727  if ((TRI->getEncodingValue(Reg) % 2) == 0)
728  return Color::Even;
729  else
730  return Color::Odd;
731 }
732 
733 // Factory function used by AArch64TargetMachine to add the pass to the passmanager.
735  return new AArch64A57FPLoadBalancing();
736 }
bool startsBefore(const Chain *Other) const
Return true if this chain starts before Other.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
MachineInstr * StartInst
The important (marker) instructions.
iterator_range< mop_iterator > uses()
Definition: MachineInstr.h:325
bool isTied() const
#define DEBUG_TYPE
F(f)
bool contains(MachineInstr *MI)
Return true if MI is a member of the chain.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const TargetRegisterInfo * getTargetRegisterInfo() const
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:75
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Reg
All possible values of the reg field in the ModR/M byte.
std::string str() const
Return a simple string representation of the chain.
MachineBasicBlock::iterator getEnd() const
Return an instruction that can be used as an iterator for the end of the chain.
ELFYAML::ELF_STO Other
Definition: ELFYAML.cpp:591
#define G(x, y, z)
Definition: MD5.cpp:52
INITIALIZE_PASS_BEGIN(AArch64A57FPLoadBalancing, DEBUG_TYPE,"AArch64 A57 FP Load-Balancing", false, false) INITIALIZE_PASS_END(AArch64A57FPLoadBalancing
void forward()
Move the internal MBB iterator and update register states.
bool isKill() const
FunctionPass * createAArch64A57FPLoadBalancing()
BitVector getRegsAvailable(const TargetRegisterClass *RC)
Return all available registers in the register class in Mask.
AArch64 A57 FP Load false
iterator insert(const ElemTy &Data)
insert - Insert a new value into the union/find set, ignoring the request if the value already exists...
bool requiresFixup() const
Return true if the group will require a fixup MOV at the end.
static cl::opt< bool > TransformAll("aarch64-a57-fp-load-balancing-force-all", cl::desc("Always modify dest registers regardless of color"), cl::init(false), cl::Hidden)
void enterBasicBlock(MachineBasicBlock *mbb)
Start tracking liveness from the begin of the specific basic block.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:267
bundle_iterator< MachineInstr, instr_iterator > iterator
MachineInstr * getKill() const
Return the "kill" instruction (as set with setKill()) or NULL.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
void clearBitsNotInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
clearBitsNotInMask - Clear a bit in this vector for every '0' bit in Mask.
Definition: BitVector.h:499
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:273
static void initializeAArch64A57FPLoadBalancingPass(PassRegistry &)
EquivalenceClasses - This represents a collection of equivalence classes and supports three efficient...
MCRegAliasIterator enumerates all registers aliasing Reg.
Represent the analysis usage information of a pass.
BitVector & reset()
Definition: BitVector.h:259
iterator_range< mop_iterator > defs()
Definition: MachineInstr.h:317
bool isKillImmutable() const
Can the Kill instruction (assuming one exists) be modified?
bool rangeOverlapsWith(const Chain &Other) const
Return true if this chain (StartInst..KillInst) overlaps with Other.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:294
AArch64 A57 FP Load Balancing
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
std::string & str()
Flushes the stream contents to the target string and returns the string's reference.
Definition: raw_ostream.h:480
Color LastColor
The "color" of LastInst.
void add(MachineInstr *MI, unsigned Idx, Color C)
Add a new instruction into the chain.
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
Color
A "color", which is either even or odd.
MachineOperand class - Representation of each machine instruction operand.
unsigned size() const
Return the number of instructions in the chain.
Promote Memory to Register
Definition: Mem2Reg.cpp:58
static bool isMla(MachineInstr *MI)
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:263
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
MachineInstr * getLast() const
Return the last instruction in the chain.
static bool clobbersPhysReg(const uint32_t *RegMask, unsigned PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
std::set< MachineInstr * > Insts
All instructions in the chain.
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Representation of each machine instruction.
Definition: MachineInstr.h:51
bool KillIsImmutable
True if KillInst cannot be modified.
A Chain is a sequence of instructions that are linked together by an accumulation operand...
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
void setReg(unsigned Reg)
Change the register this operand corresponds to.
#define I(x, y, z)
Definition: MD5.cpp:54
unsigned getReg() const
getReg - Returns the register number.
unsigned StartInstIdx
The index, from the start of the basic block, that each marker appears.
void setKill(MachineInstr *MI, unsigned Idx, bool Immutable)
Inform the chain that its last active register (the dest register of LastInst) is killed by MI with n...
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:465
static cl::opt< unsigned > OverrideBalance("aarch64-a57-fp-load-balancing-override", cl::desc("Ignore balance information, always return ""(1: Even, 2: Odd)."), cl::init(0), cl::Hidden)
Color getPreferredColor()
Return the preferred color of this chain.
#define DEBUG(X)
Definition: Debug.h:92
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
static bool isMul(MachineInstr *MI)
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
Definition: PassRegistry.h:41
MachineInstr * getStart() const
Return the first instruction in the chain.
Chain(MachineInstr *MI, unsigned Idx, Color C)