LLVM  6.0.0svn
HexagonVLIWPacketizer.cpp
Go to the documentation of this file.
1 //===- HexagonPacketizer.cpp - VLIW packetizer ----------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements a simple VLIW packetizer using DFA. The packetizer works on
11 // machine basic blocks. For each instruction I in BB, the packetizer consults
12 // the DFA to see if machine resources are available to execute I. If so, the
13 // packetizer checks if I depends on any instruction J in the current packet.
14 // If no dependency is found, I is added to current packet and machine resource
15 // is marked as taken. If any dependency is found, a target API call is made to
16 // prune the dependence.
17 //
18 //===----------------------------------------------------------------------===//
19 
20 #include "HexagonVLIWPacketizer.h"
21 #include "Hexagon.h"
22 #include "HexagonInstrInfo.h"
23 #include "HexagonRegisterInfo.h"
24 #include "HexagonSubtarget.h"
25 #include "llvm/ADT/BitVector.h"
26 #include "llvm/ADT/DenseSet.h"
27 #include "llvm/ADT/STLExtras.h"
41 #include "llvm/IR/DebugLoc.h"
42 #include "llvm/MC/MCInstrDesc.h"
43 #include "llvm/Pass.h"
45 #include "llvm/Support/Debug.h"
48 #include <cassert>
49 #include <cstdint>
50 #include <iterator>
51 
52 using namespace llvm;
53 
54 #define DEBUG_TYPE "packets"
55 
56 static cl::opt<bool> DisablePacketizer("disable-packetizer", cl::Hidden,
57  cl::ZeroOrMore, cl::init(false),
58  cl::desc("Disable Hexagon packetizer pass"));
59 
60 static cl::opt<bool> PacketizeVolatiles("hexagon-packetize-volatiles",
62  cl::desc("Allow non-solo packetization of volatile memory references"));
63 
64 static cl::opt<bool> EnableGenAllInsnClass("enable-gen-insn", cl::init(false),
65  cl::Hidden, cl::ZeroOrMore, cl::desc("Generate all instruction with TC"));
66 
67 static cl::opt<bool> DisableVecDblNVStores("disable-vecdbl-nv-stores",
69  cl::desc("Disable vector double new-value-stores"));
70 
72 
73 namespace llvm {
74 
77 
78 } // end namespace llvm
79 
80 namespace {
81 
82  class HexagonPacketizer : public MachineFunctionPass {
83  public:
84  static char ID;
85 
86  HexagonPacketizer() : MachineFunctionPass(ID) {}
87 
88  void getAnalysisUsage(AnalysisUsage &AU) const override {
89  AU.setPreservesCFG();
97  }
98 
99  StringRef getPassName() const override { return "Hexagon Packetizer"; }
100  bool runOnMachineFunction(MachineFunction &Fn) override;
101 
102  MachineFunctionProperties getRequiredProperties() const override {
105  }
106 
107  private:
108  const HexagonInstrInfo *HII;
109  const HexagonRegisterInfo *HRI;
110  };
111 
112 } // end anonymous namespace
113 
114 char HexagonPacketizer::ID = 0;
115 
116 INITIALIZE_PASS_BEGIN(HexagonPacketizer, "hexagon-packetizer",
117  "Hexagon Packetizer", false, false)
122 INITIALIZE_PASS_END(HexagonPacketizer, "hexagon-packetizer",
123  "Hexagon Packetizer", false, false)
124 
126  MachineLoopInfo &MLI, AliasAnalysis *AA,
128  : VLIWPacketizerList(MF, MLI, AA), MBPI(MBPI), MLI(&MLI) {
129  HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
130  HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
131 
132  addMutation(llvm::make_unique<HexagonSubtarget::UsrOverflowMutation>());
133  addMutation(llvm::make_unique<HexagonSubtarget::HVXMemLatencyMutation>());
134  addMutation(llvm::make_unique<HexagonSubtarget::BankConflictMutation>());
135 }
136 
137 // Check if FirstI modifies a register that SecondI reads.
138 static bool hasWriteToReadDep(const MachineInstr &FirstI,
139  const MachineInstr &SecondI,
140  const TargetRegisterInfo *TRI) {
141  for (auto &MO : FirstI.operands()) {
142  if (!MO.isReg() || !MO.isDef())
143  continue;
144  unsigned R = MO.getReg();
145  if (SecondI.readsRegister(R, TRI))
146  return true;
147  }
148  return false;
149 }
150 
151 
153  MachineBasicBlock::iterator BundleIt, bool Before) {
155  if (Before)
156  InsertPt = BundleIt.getInstrIterator();
157  else
158  InsertPt = std::next(BundleIt).getInstrIterator();
159 
160  MachineBasicBlock &B = *MI.getParent();
161  // The instruction should at least be bundled with the preceding instruction
162  // (there will always be one, i.e. BUNDLE, if nothing else).
164  if (MI.isBundledWithSucc()) {
167  } else {
168  // If it's not bundled with the successor (i.e. it is the last one
169  // in the bundle), then we can simply unbundle it from the predecessor,
170  // which will take care of updating the predecessor's flag.
171  MI.unbundleFromPred();
172  }
173  B.splice(InsertPt, &B, MI.getIterator());
174 
175  // Get the size of the bundle without asserting.
178  unsigned Size = 0;
179  for (++I; I != E && I->isBundledWithPred(); ++I)
180  ++Size;
181 
182  // If there are still two or more instructions, then there is nothing
183  // else to be done.
184  if (Size > 1)
185  return BundleIt;
186 
187  // Otherwise, extract the single instruction out and delete the bundle.
188  MachineBasicBlock::iterator NextIt = std::next(BundleIt);
189  MachineInstr &SingleI = *BundleIt->getNextNode();
190  SingleI.unbundleFromPred();
191  assert(!SingleI.isBundledWithSucc());
192  BundleIt->eraseFromParent();
193  return NextIt;
194 }
195 
196 bool HexagonPacketizer::runOnMachineFunction(MachineFunction &MF) {
197  if (DisablePacketizer || skipFunction(*MF.getFunction()))
198  return false;
199 
200  HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
201  HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
202  auto &MLI = getAnalysis<MachineLoopInfo>();
203  auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
204  auto *MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
205 
207  HII->genAllInsnTimingClasses(MF);
208 
209  // Instantiate the packetizer.
210  HexagonPacketizerList Packetizer(MF, MLI, AA, MBPI);
211 
212  // DFA state table should not be empty.
213  assert(Packetizer.getResourceTracker() && "Empty DFA table!");
214 
215  // Loop over all basic blocks and remove KILL pseudo-instructions
216  // These instructions confuse the dependence analysis. Consider:
217  // D0 = ... (Insn 0)
218  // R0 = KILL R0, D0 (Insn 1)
219  // R0 = ... (Insn 2)
220  // Here, Insn 1 will result in the dependence graph not emitting an output
221  // dependence between Insn 0 and Insn 2. This can lead to incorrect
222  // packetization
223  for (auto &MB : MF) {
224  auto End = MB.end();
225  auto MI = MB.begin();
226  while (MI != End) {
227  auto NextI = std::next(MI);
228  if (MI->isKill()) {
229  MB.erase(MI);
230  End = MB.end();
231  }
232  MI = NextI;
233  }
234  }
235 
236  // Loop over all of the basic blocks.
237  for (auto &MB : MF) {
238  auto Begin = MB.begin(), End = MB.end();
239  while (Begin != End) {
240  // Find the first non-boundary starting from the end of the last
241  // scheduling region.
242  MachineBasicBlock::iterator RB = Begin;
243  while (RB != End && HII->isSchedulingBoundary(*RB, &MB, MF))
244  ++RB;
245  // Find the first boundary starting from the beginning of the new
246  // region.
248  while (RE != End && !HII->isSchedulingBoundary(*RE, &MB, MF))
249  ++RE;
250  // Add the scheduling boundary if it's not block end.
251  if (RE != End)
252  ++RE;
253  // If RB == End, then RE == End.
254  if (RB != End)
255  Packetizer.PacketizeMIs(&MB, RB, RE);
256 
257  Begin = RE;
258  }
259  }
260 
261  Packetizer.unpacketizeSoloInstrs(MF);
262  return true;
263 }
264 
265 // Reserve resources for a constant extender. Trigger an assertion if the
266 // reservation fails.
268  if (!tryAllocateResourcesForConstExt(true))
269  llvm_unreachable("Resources not available");
270 }
271 
273  return tryAllocateResourcesForConstExt(false);
274 }
275 
276 // Allocate resources (i.e. 4 bytes) for constant extender. If succeeded,
277 // return true, otherwise, return false.
279  auto *ExtMI = MF.CreateMachineInstr(HII->get(Hexagon::A4_ext), DebugLoc());
280  bool Avail = ResourceTracker->canReserveResources(*ExtMI);
281  if (Reserve && Avail)
282  ResourceTracker->reserveResources(*ExtMI);
283  MF.DeleteMachineInstr(ExtMI);
284  return Avail;
285 }
286 
288  SDep::Kind DepType, unsigned DepReg) {
289  // Check for LR dependence.
290  if (DepReg == HRI->getRARegister())
291  return true;
292 
293  if (HII->isDeallocRet(MI))
294  if (DepReg == HRI->getFrameRegister() || DepReg == HRI->getStackRegister())
295  return true;
296 
297  // Call-like instructions can be packetized with preceding instructions
298  // that define registers implicitly used or modified by the call. Explicit
299  // uses are still prohibited, as in the case of indirect calls:
300  // r0 = ...
301  // J2_jumpr r0
302  if (DepType == SDep::Data) {
303  for (const MachineOperand MO : MI.operands())
304  if (MO.isReg() && MO.getReg() == DepReg && !MO.isImplicit())
305  return true;
306  }
307 
308  return false;
309 }
310 
311 static bool isRegDependence(const SDep::Kind DepType) {
312  return DepType == SDep::Data || DepType == SDep::Anti ||
313  DepType == SDep::Output;
314 }
315 
316 static bool isDirectJump(const MachineInstr &MI) {
317  return MI.getOpcode() == Hexagon::J2_jump;
318 }
319 
320 static bool isSchedBarrier(const MachineInstr &MI) {
321  switch (MI.getOpcode()) {
322  case Hexagon::Y2_barrier:
323  return true;
324  }
325  return false;
326 }
327 
328 static bool isControlFlow(const MachineInstr &MI) {
329  return MI.getDesc().isTerminator() || MI.getDesc().isCall();
330 }
331 
332 /// Returns true if the instruction modifies a callee-saved register.
334  const TargetRegisterInfo *TRI) {
335  const MachineFunction &MF = *MI.getParent()->getParent();
336  for (auto *CSR = TRI->getCalleeSavedRegs(&MF); CSR && *CSR; ++CSR)
337  if (MI.modifiesRegister(*CSR, TRI))
338  return true;
339  return false;
340 }
341 
342 // Returns true if an instruction can be promoted to .new predicate or
343 // new-value store.
345  const TargetRegisterClass *NewRC) {
346  // Vector stores can be predicated, and can be new-value stores, but
347  // they cannot be predicated on a .new predicate value.
348  if (NewRC == &Hexagon::PredRegsRegClass) {
349  if (HII->isHVXVec(MI) && MI.mayStore())
350  return false;
351  return HII->isPredicated(MI) && HII->getDotNewPredOp(MI, nullptr) > 0;
352  }
353  // If the class is not PredRegs, it could only apply to new-value stores.
354  return HII->mayBeNewStore(MI);
355 }
356 
357 // Promote an instructiont to its .cur form.
358 // At this time, we have already made a call to canPromoteToDotCur and made
359 // sure that it can *indeed* be promoted.
362  const TargetRegisterClass* RC) {
363  assert(DepType == SDep::Data);
364  int CurOpcode = HII->getDotCurOp(MI);
365  MI.setDesc(HII->get(CurOpcode));
366  return true;
367 }
368 
370  MachineInstr *MI = nullptr;
371  for (auto BI : CurrentPacketMIs) {
372  DEBUG(dbgs() << "Cleanup packet has "; BI->dump(););
373  if (HII->isDotCurInst(*BI)) {
374  MI = BI;
375  continue;
376  }
377  if (MI) {
378  for (auto &MO : BI->operands())
379  if (MO.isReg() && MO.getReg() == MI->getOperand(0).getReg())
380  return;
381  }
382  }
383  if (!MI)
384  return;
385  // We did not find a use of the CUR, so de-cur it.
386  MI->setDesc(HII->get(HII->getNonDotCurOp(*MI)));
387  DEBUG(dbgs() << "Demoted CUR "; MI->dump(););
388 }
389 
390 // Check to see if an instruction can be dot cur.
392  const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
393  const TargetRegisterClass *RC) {
394  if (!HII->isHVXVec(MI))
395  return false;
396  if (!HII->isHVXVec(*MII))
397  return false;
398 
399  // Already a dot new instruction.
400  if (HII->isDotCurInst(MI) && !HII->mayBeCurLoad(MI))
401  return false;
402 
403  if (!HII->mayBeCurLoad(MI))
404  return false;
405 
406  // The "cur value" cannot come from inline asm.
407  if (PacketSU->getInstr()->isInlineAsm())
408  return false;
409 
410  // Make sure candidate instruction uses cur.
411  DEBUG(dbgs() << "Can we DOT Cur Vector MI\n";
412  MI.dump();
413  dbgs() << "in packet\n";);
414  MachineInstr &MJ = *MII;
415  DEBUG({
416  dbgs() << "Checking CUR against ";
417  MJ.dump();
418  });
419  unsigned DestReg = MI.getOperand(0).getReg();
420  bool FoundMatch = false;
421  for (auto &MO : MJ.operands())
422  if (MO.isReg() && MO.getReg() == DestReg)
423  FoundMatch = true;
424  if (!FoundMatch)
425  return false;
426 
427  // Check for existing uses of a vector register within the packet which
428  // would be affected by converting a vector load into .cur formt.
429  for (auto BI : CurrentPacketMIs) {
430  DEBUG(dbgs() << "packet has "; BI->dump(););
431  if (BI->readsRegister(DepReg, MF.getSubtarget().getRegisterInfo()))
432  return false;
433  }
434 
435  DEBUG(dbgs() << "Can Dot CUR MI\n"; MI.dump(););
436  // We can convert the opcode into a .cur.
437  return true;
438 }
439 
440 // Promote an instruction to its .new form. At this time, we have already
441 // made a call to canPromoteToDotNew and made sure that it can *indeed* be
442 // promoted.
445  const TargetRegisterClass* RC) {
446  assert(DepType == SDep::Data);
447  int NewOpcode;
448  if (RC == &Hexagon::PredRegsRegClass)
449  NewOpcode = HII->getDotNewPredOp(MI, MBPI);
450  else
451  NewOpcode = HII->getDotNewOp(MI);
452  MI.setDesc(HII->get(NewOpcode));
453  return true;
454 }
455 
457  int NewOpcode = HII->getDotOldOp(MI);
458  MI.setDesc(HII->get(NewOpcode));
459  return true;
460 }
461 
463  unsigned Opc = MI.getOpcode();
464  switch (Opc) {
465  case Hexagon::S2_storerd_io:
466  case Hexagon::S2_storeri_io:
467  case Hexagon::S2_storerh_io:
468  case Hexagon::S2_storerb_io:
469  break;
470  default:
471  llvm_unreachable("Unexpected instruction");
472  }
473  unsigned FrameSize = MF.getFrameInfo().getStackSize();
474  MachineOperand &Off = MI.getOperand(1);
475  int64_t NewOff = Off.getImm() - (FrameSize + HEXAGON_LRFP_SIZE);
476  if (HII->isValidOffset(Opc, NewOff, HRI)) {
477  Off.setImm(NewOff);
478  return true;
479  }
480  return false;
481 }
482 
484  unsigned Opc = MI.getOpcode();
485  switch (Opc) {
486  case Hexagon::S2_storerd_io:
487  case Hexagon::S2_storeri_io:
488  case Hexagon::S2_storerh_io:
489  case Hexagon::S2_storerb_io:
490  break;
491  default:
492  llvm_unreachable("Unexpected instruction");
493  }
494  unsigned FrameSize = MF.getFrameInfo().getStackSize();
495  MachineOperand &Off = MI.getOperand(1);
496  Off.setImm(Off.getImm() + FrameSize + HEXAGON_LRFP_SIZE);
497 }
498 
499 /// Return true if we can update the offset in MI so that MI and MJ
500 /// can be packetized together.
502  assert(SUI->getInstr() && SUJ->getInstr());
503  MachineInstr &MI = *SUI->getInstr();
504  MachineInstr &MJ = *SUJ->getInstr();
505 
506  unsigned BPI, OPI;
507  if (!HII->getBaseAndOffsetPosition(MI, BPI, OPI))
508  return false;
509  unsigned BPJ, OPJ;
510  if (!HII->getBaseAndOffsetPosition(MJ, BPJ, OPJ))
511  return false;
512  unsigned Reg = MI.getOperand(BPI).getReg();
513  if (Reg != MJ.getOperand(BPJ).getReg())
514  return false;
515  // Make sure that the dependences do not restrict adding MI to the packet.
516  // That is, ignore anti dependences, and make sure the only data dependence
517  // involves the specific register.
518  for (const auto &PI : SUI->Preds)
519  if (PI.getKind() != SDep::Anti &&
520  (PI.getKind() != SDep::Data || PI.getReg() != Reg))
521  return false;
522  int Incr;
523  if (!HII->getIncrementValue(MJ, Incr))
524  return false;
525 
526  int64_t Offset = MI.getOperand(OPI).getImm();
527  MI.getOperand(OPI).setImm(Offset + Incr);
528  ChangedOffset = Offset;
529  return true;
530 }
531 
532 /// Undo the changed offset. This is needed if the instruction cannot be
533 /// added to the current packet due to a different instruction.
535  unsigned BP, OP;
536  if (!HII->getBaseAndOffsetPosition(MI, BP, OP))
537  llvm_unreachable("Unable to find base and offset operands.");
538  MI.getOperand(OP).setImm(ChangedOffset);
539 }
540 
545 };
546 
547 /// Returns true if an instruction is predicated on p0 and false if it's
548 /// predicated on !p0.
550  const HexagonInstrInfo *HII) {
551  if (!HII->isPredicated(MI))
552  return PK_Unknown;
553  if (HII->isPredicatedTrue(MI))
554  return PK_True;
555  return PK_False;
556 }
557 
559  const HexagonInstrInfo *HII) {
560  assert(HII->isPostIncrement(MI) && "Not a post increment operation.");
561 #ifndef NDEBUG
562  // Post Increment means duplicates. Use dense map to find duplicates in the
563  // list. Caution: Densemap initializes with the minimum of 64 buckets,
564  // whereas there are at most 5 operands in the post increment.
565  DenseSet<unsigned> DefRegsSet;
566  for (auto &MO : MI.operands())
567  if (MO.isReg() && MO.isDef())
568  DefRegsSet.insert(MO.getReg());
569 
570  for (auto &MO : MI.operands())
571  if (MO.isReg() && MO.isUse() && DefRegsSet.count(MO.getReg()))
572  return MO;
573 #else
574  if (MI.mayLoad()) {
575  const MachineOperand &Op1 = MI.getOperand(1);
576  // The 2nd operand is always the post increment operand in load.
577  assert(Op1.isReg() && "Post increment operand has be to a register.");
578  return Op1;
579  }
580  if (MI.getDesc().mayStore()) {
581  const MachineOperand &Op0 = MI.getOperand(0);
582  // The 1st operand is always the post increment operand in store.
583  assert(Op0.isReg() && "Post increment operand has be to a register.");
584  return Op0;
585  }
586 #endif
587  // we should never come here.
588  llvm_unreachable("mayLoad or mayStore not set for Post Increment operation");
589 }
590 
591 // Get the value being stored.
593  // value being stored is always the last operand.
594  return MI.getOperand(MI.getNumOperands()-1);
595 }
596 
597 static bool isLoadAbsSet(const MachineInstr &MI) {
598  unsigned Opc = MI.getOpcode();
599  switch (Opc) {
600  case Hexagon::L4_loadrd_ap:
601  case Hexagon::L4_loadrb_ap:
602  case Hexagon::L4_loadrh_ap:
603  case Hexagon::L4_loadrub_ap:
604  case Hexagon::L4_loadruh_ap:
605  case Hexagon::L4_loadri_ap:
606  return true;
607  }
608  return false;
609 }
610 
612  assert(isLoadAbsSet(MI));
613  return MI.getOperand(1);
614 }
615 
616 // Can be new value store?
617 // Following restrictions are to be respected in convert a store into
618 // a new value store.
619 // 1. If an instruction uses auto-increment, its address register cannot
620 // be a new-value register. Arch Spec 5.4.2.1
621 // 2. If an instruction uses absolute-set addressing mode, its address
622 // register cannot be a new-value register. Arch Spec 5.4.2.1.
623 // 3. If an instruction produces a 64-bit result, its registers cannot be used
624 // as new-value registers. Arch Spec 5.4.2.2.
625 // 4. If the instruction that sets the new-value register is conditional, then
626 // the instruction that uses the new-value register must also be conditional,
627 // and both must always have their predicates evaluate identically.
628 // Arch Spec 5.4.2.3.
629 // 5. There is an implied restriction that a packet cannot have another store,
630 // if there is a new value store in the packet. Corollary: if there is
631 // already a store in a packet, there can not be a new value store.
632 // Arch Spec: 3.4.4.2
634  const MachineInstr &PacketMI, unsigned DepReg) {
635  // Make sure we are looking at the store, that can be promoted.
636  if (!HII->mayBeNewStore(MI))
637  return false;
638 
639  // Make sure there is dependency and can be new value'd.
640  const MachineOperand &Val = getStoreValueOperand(MI);
641  if (Val.isReg() && Val.getReg() != DepReg)
642  return false;
643 
644  const MCInstrDesc& MCID = PacketMI.getDesc();
645 
646  // First operand is always the result.
647  const TargetRegisterClass *PacketRC = HII->getRegClass(MCID, 0, HRI, MF);
648  // Double regs can not feed into new value store: PRM section: 5.4.2.2.
649  if (PacketRC == &Hexagon::DoubleRegsRegClass)
650  return false;
651 
652  // New-value stores are of class NV (slot 0), dual stores require class ST
653  // in slot 0 (PRM 5.5).
654  for (auto I : CurrentPacketMIs) {
655  SUnit *PacketSU = MIToSUnit.find(I)->second;
656  if (PacketSU->getInstr()->mayStore())
657  return false;
658  }
659 
660  // Make sure it's NOT the post increment register that we are going to
661  // new value.
662  if (HII->isPostIncrement(MI) &&
663  getPostIncrementOperand(MI, HII).getReg() == DepReg) {
664  return false;
665  }
666 
667  if (HII->isPostIncrement(PacketMI) && PacketMI.mayLoad() &&
668  getPostIncrementOperand(PacketMI, HII).getReg() == DepReg) {
669  // If source is post_inc, or absolute-set addressing, it can not feed
670  // into new value store
671  // r3 = memw(r2++#4)
672  // memw(r30 + #-1404) = r2.new -> can not be new value store
673  // arch spec section: 5.4.2.1.
674  return false;
675  }
676 
677  if (isLoadAbsSet(PacketMI) && getAbsSetOperand(PacketMI).getReg() == DepReg)
678  return false;
679 
680  // If the source that feeds the store is predicated, new value store must
681  // also be predicated.
682  if (HII->isPredicated(PacketMI)) {
683  if (!HII->isPredicated(MI))
684  return false;
685 
686  // Check to make sure that they both will have their predicates
687  // evaluate identically.
688  unsigned predRegNumSrc = 0;
689  unsigned predRegNumDst = 0;
690  const TargetRegisterClass* predRegClass = nullptr;
691 
692  // Get predicate register used in the source instruction.
693  for (auto &MO : PacketMI.operands()) {
694  if (!MO.isReg())
695  continue;
696  predRegNumSrc = MO.getReg();
697  predRegClass = HRI->getMinimalPhysRegClass(predRegNumSrc);
698  if (predRegClass == &Hexagon::PredRegsRegClass)
699  break;
700  }
701  assert((predRegClass == &Hexagon::PredRegsRegClass) &&
702  "predicate register not found in a predicated PacketMI instruction");
703 
704  // Get predicate register used in new-value store instruction.
705  for (auto &MO : MI.operands()) {
706  if (!MO.isReg())
707  continue;
708  predRegNumDst = MO.getReg();
709  predRegClass = HRI->getMinimalPhysRegClass(predRegNumDst);
710  if (predRegClass == &Hexagon::PredRegsRegClass)
711  break;
712  }
713  assert((predRegClass == &Hexagon::PredRegsRegClass) &&
714  "predicate register not found in a predicated MI instruction");
715 
716  // New-value register producer and user (store) need to satisfy these
717  // constraints:
718  // 1) Both instructions should be predicated on the same register.
719  // 2) If producer of the new-value register is .new predicated then store
720  // should also be .new predicated and if producer is not .new predicated
721  // then store should not be .new predicated.
722  // 3) Both new-value register producer and user should have same predicate
723  // sense, i.e, either both should be negated or both should be non-negated.
724  if (predRegNumDst != predRegNumSrc ||
725  HII->isDotNewInst(PacketMI) != HII->isDotNewInst(MI) ||
726  getPredicateSense(MI, HII) != getPredicateSense(PacketMI, HII))
727  return false;
728  }
729 
730  // Make sure that other than the new-value register no other store instruction
731  // register has been modified in the same packet. Predicate registers can be
732  // modified by they should not be modified between the producer and the store
733  // instruction as it will make them both conditional on different values.
734  // We already know this to be true for all the instructions before and
735  // including PacketMI. Howerver, we need to perform the check for the
736  // remaining instructions in the packet.
737 
738  unsigned StartCheck = 0;
739 
740  for (auto I : CurrentPacketMIs) {
741  SUnit *TempSU = MIToSUnit.find(I)->second;
742  MachineInstr &TempMI = *TempSU->getInstr();
743 
744  // Following condition is true for all the instructions until PacketMI is
745  // reached (StartCheck is set to 0 before the for loop).
746  // StartCheck flag is 1 for all the instructions after PacketMI.
747  if (&TempMI != &PacketMI && !StartCheck) // Start processing only after
748  continue; // encountering PacketMI.
749 
750  StartCheck = 1;
751  if (&TempMI == &PacketMI) // We don't want to check PacketMI for dependence.
752  continue;
753 
754  for (auto &MO : MI.operands())
755  if (MO.isReg() && TempSU->getInstr()->modifiesRegister(MO.getReg(), HRI))
756  return false;
757  }
758 
759  // Make sure that for non-POST_INC stores:
760  // 1. The only use of reg is DepReg and no other registers.
761  // This handles V4 base+index registers.
762  // The following store can not be dot new.
763  // Eg. r0 = add(r0, #3)
764  // memw(r1+r0<<#2) = r0
765  if (!HII->isPostIncrement(MI)) {
766  for (unsigned opNum = 0; opNum < MI.getNumOperands()-1; opNum++) {
767  const MachineOperand &MO = MI.getOperand(opNum);
768  if (MO.isReg() && MO.getReg() == DepReg)
769  return false;
770  }
771  }
772 
773  // If data definition is because of implicit definition of the register,
774  // do not newify the store. Eg.
775  // %R9<def> = ZXTH %R12, %D6<imp-use>, %R12<imp-def>
776  // S2_storerh_io %R8, 2, %R12<kill>; mem:ST2[%scevgep343]
777  for (auto &MO : PacketMI.operands()) {
778  if (MO.isRegMask() && MO.clobbersPhysReg(DepReg))
779  return false;
780  if (!MO.isReg() || !MO.isDef() || !MO.isImplicit())
781  continue;
782  unsigned R = MO.getReg();
783  if (R == DepReg || HRI->isSuperRegister(DepReg, R))
784  return false;
785  }
786 
787  // Handle imp-use of super reg case. There is a target independent side
788  // change that should prevent this situation but I am handling it for
789  // just-in-case. For example, we cannot newify R2 in the following case:
790  // %R3<def> = A2_tfrsi 0;
791  // S2_storeri_io %R0<kill>, 0, %R2<kill>, %D1<imp-use,kill>;
792  for (auto &MO : MI.operands()) {
793  if (MO.isReg() && MO.isUse() && MO.isImplicit() && MO.getReg() == DepReg)
794  return false;
795  }
796 
797  // Can be dot new store.
798  return true;
799 }
800 
801 // Can this MI to promoted to either new value store or new value jump.
803  const SUnit *PacketSU, unsigned DepReg,
805  if (!HII->mayBeNewStore(MI))
806  return false;
807 
808  // Check to see the store can be new value'ed.
809  MachineInstr &PacketMI = *PacketSU->getInstr();
810  if (canPromoteToNewValueStore(MI, PacketMI, DepReg))
811  return true;
812 
813  // Check to see the compare/jump can be new value'ed.
814  // This is done as a pass on its own. Don't need to check it here.
815  return false;
816 }
817 
818 static bool isImplicitDependency(const MachineInstr &I, bool CheckDef,
819  unsigned DepReg) {
820  for (auto &MO : I.operands()) {
821  if (CheckDef && MO.isRegMask() && MO.clobbersPhysReg(DepReg))
822  return true;
823  if (!MO.isReg() || MO.getReg() != DepReg || !MO.isImplicit())
824  continue;
825  if (CheckDef == MO.isDef())
826  return true;
827  }
828  return false;
829 }
830 
831 // Check to see if an instruction can be dot new
832 // There are three kinds.
833 // 1. dot new on predicate - V2/V3/V4
834 // 2. dot new on stores NV/ST - V4
835 // 3. dot new on jump NV/J - V4 -- This is generated in a pass.
837  const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
838  const TargetRegisterClass* RC) {
839  // Already a dot new instruction.
840  if (HII->isDotNewInst(MI) && !HII->mayBeNewStore(MI))
841  return false;
842 
843  if (!isNewifiable(MI, RC))
844  return false;
845 
846  const MachineInstr &PI = *PacketSU->getInstr();
847 
848  // The "new value" cannot come from inline asm.
849  if (PI.isInlineAsm())
850  return false;
851 
852  // IMPLICIT_DEFs won't materialize as real instructions, so .new makes no
853  // sense.
854  if (PI.isImplicitDef())
855  return false;
856 
857  // If dependency is trough an implicitly defined register, we should not
858  // newify the use.
859  if (isImplicitDependency(PI, true, DepReg) ||
860  isImplicitDependency(MI, false, DepReg))
861  return false;
862 
863  const MCInstrDesc& MCID = PI.getDesc();
864  const TargetRegisterClass *VecRC = HII->getRegClass(MCID, 0, HRI, MF);
865  if (DisableVecDblNVStores && VecRC == &Hexagon::HvxWRRegClass)
866  return false;
867 
868  // predicate .new
869  if (RC == &Hexagon::PredRegsRegClass)
870  return HII->predCanBeUsedAsDotNew(PI, DepReg);
871 
872  if (RC != &Hexagon::PredRegsRegClass && !HII->mayBeNewStore(MI))
873  return false;
874 
875  // Create a dot new machine instruction to see if resources can be
876  // allocated. If not, bail out now.
877  int NewOpcode = HII->getDotNewOp(MI);
878  const MCInstrDesc &D = HII->get(NewOpcode);
879  MachineInstr *NewMI = MF.CreateMachineInstr(D, DebugLoc());
880  bool ResourcesAvailable = ResourceTracker->canReserveResources(*NewMI);
881  MF.DeleteMachineInstr(NewMI);
882  if (!ResourcesAvailable)
883  return false;
884 
885  // New Value Store only. New Value Jump generated as a separate pass.
886  if (!canPromoteToNewValue(MI, PacketSU, DepReg, MII))
887  return false;
888 
889  return true;
890 }
891 
892 // Go through the packet instructions and search for an anti dependency between
893 // them and DepReg from MI. Consider this case:
894 // Trying to add
895 // a) %R1<def> = TFRI_cdNotPt %P3, 2
896 // to this packet:
897 // {
898 // b) %P0<def> = C2_or %P3<kill>, %P0<kill>
899 // c) %P3<def> = C2_tfrrp %R23
900 // d) %R1<def> = C2_cmovenewit %P3, 4
901 // }
902 // The P3 from a) and d) will be complements after
903 // a)'s P3 is converted to .new form
904 // Anti-dep between c) and b) is irrelevant for this case
906  unsigned DepReg) {
907  SUnit *PacketSUDep = MIToSUnit.find(&MI)->second;
908 
909  for (auto I : CurrentPacketMIs) {
910  // We only care for dependencies to predicated instructions
911  if (!HII->isPredicated(*I))
912  continue;
913 
914  // Scheduling Unit for current insn in the packet
915  SUnit *PacketSU = MIToSUnit.find(I)->second;
916 
917  // Look at dependencies between current members of the packet and
918  // predicate defining instruction MI. Make sure that dependency is
919  // on the exact register we care about.
920  if (PacketSU->isSucc(PacketSUDep)) {
921  for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
922  auto &Dep = PacketSU->Succs[i];
923  if (Dep.getSUnit() == PacketSUDep && Dep.getKind() == SDep::Anti &&
924  Dep.getReg() == DepReg)
925  return true;
926  }
927  }
928  }
929 
930  return false;
931 }
932 
933 /// Gets the predicate register of a predicated instruction.
935  const HexagonInstrInfo *QII) {
936  /// We use the following rule: The first predicate register that is a use is
937  /// the predicate register of a predicated instruction.
938  assert(QII->isPredicated(MI) && "Must be predicated instruction");
939 
940  for (auto &Op : MI.operands()) {
941  if (Op.isReg() && Op.getReg() && Op.isUse() &&
942  Hexagon::PredRegsRegClass.contains(Op.getReg()))
943  return Op.getReg();
944  }
945 
946  llvm_unreachable("Unknown instruction operand layout");
947  return 0;
948 }
949 
950 // Given two predicated instructions, this function detects whether
951 // the predicates are complements.
953  MachineInstr &MI2) {
954  // If we don't know the predicate sense of the instructions bail out early, we
955  // need it later.
956  if (getPredicateSense(MI1, HII) == PK_Unknown ||
957  getPredicateSense(MI2, HII) == PK_Unknown)
958  return false;
959 
960  // Scheduling unit for candidate.
961  SUnit *SU = MIToSUnit[&MI1];
962 
963  // One corner case deals with the following scenario:
964  // Trying to add
965  // a) %R24<def> = A2_tfrt %P0, %R25
966  // to this packet:
967  // {
968  // b) %R25<def> = A2_tfrf %P0, %R24
969  // c) %P0<def> = C2_cmpeqi %R26, 1
970  // }
971  //
972  // On general check a) and b) are complements, but presence of c) will
973  // convert a) to .new form, and then it is not a complement.
974  // We attempt to detect it by analyzing existing dependencies in the packet.
975 
976  // Analyze relationships between all existing members of the packet.
977  // Look for Anti dependecy on the same predicate reg as used in the
978  // candidate.
979  for (auto I : CurrentPacketMIs) {
980  // Scheduling Unit for current insn in the packet.
981  SUnit *PacketSU = MIToSUnit.find(I)->second;
982 
983  // If this instruction in the packet is succeeded by the candidate...
984  if (PacketSU->isSucc(SU)) {
985  for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
986  auto Dep = PacketSU->Succs[i];
987  // The corner case exist when there is true data dependency between
988  // candidate and one of current packet members, this dep is on
989  // predicate reg, and there already exist anti dep on the same pred in
990  // the packet.
991  if (Dep.getSUnit() == SU && Dep.getKind() == SDep::Data &&
992  Hexagon::PredRegsRegClass.contains(Dep.getReg())) {
993  // Here I know that I is predicate setting instruction with true
994  // data dep to candidate on the register we care about - c) in the
995  // above example. Now I need to see if there is an anti dependency
996  // from c) to any other instruction in the same packet on the pred
997  // reg of interest.
998  if (restrictingDepExistInPacket(*I, Dep.getReg()))
999  return false;
1000  }
1001  }
1002  }
1003  }
1004 
1005  // If the above case does not apply, check regular complement condition.
1006  // Check that the predicate register is the same and that the predicate
1007  // sense is different We also need to differentiate .old vs. .new: !p0
1008  // is not complementary to p0.new.
1009  unsigned PReg1 = getPredicatedRegister(MI1, HII);
1010  unsigned PReg2 = getPredicatedRegister(MI2, HII);
1011  return PReg1 == PReg2 &&
1012  Hexagon::PredRegsRegClass.contains(PReg1) &&
1013  Hexagon::PredRegsRegClass.contains(PReg2) &&
1014  getPredicateSense(MI1, HII) != getPredicateSense(MI2, HII) &&
1015  HII->isDotNewInst(MI1) == HII->isDotNewInst(MI2);
1016 }
1017 
1018 // Initialize packetizer flags.
1020  Dependence = false;
1021  PromotedToDotNew = false;
1022  GlueToNewValueJump = false;
1023  GlueAllocframeStore = false;
1024  FoundSequentialDependence = false;
1025  ChangedOffset = INT64_MAX;
1026 }
1027 
1028 // Ignore bundling of pseudo instructions.
1030  const MachineBasicBlock *) {
1031  if (MI.isDebugValue())
1032  return true;
1033 
1034  if (MI.isCFIInstruction())
1035  return false;
1036 
1037  // We must print out inline assembly.
1038  if (MI.isInlineAsm())
1039  return false;
1040 
1041  if (MI.isImplicitDef())
1042  return false;
1043 
1044  // We check if MI has any functional units mapped to it. If it doesn't,
1045  // we ignore the instruction.
1046  const MCInstrDesc& TID = MI.getDesc();
1047  auto *IS = ResourceTracker->getInstrItins()->beginStage(TID.getSchedClass());
1048  unsigned FuncUnits = IS->getUnits();
1049  return !FuncUnits;
1050 }
1051 
1053  if (MI.isEHLabel() || MI.isCFIInstruction())
1054  return true;
1055 
1056  // Consider inline asm to not be a solo instruction by default.
1057  // Inline asm will be put in a packet temporarily, but then it will be
1058  // removed, and placed outside of the packet (before or after, depending
1059  // on dependencies). This is to reduce the impact of inline asm as a
1060  // "packet splitting" instruction.
1061  if (MI.isInlineAsm() && !ScheduleInlineAsm)
1062  return true;
1063 
1064  // From Hexagon V4 Programmer's Reference Manual 3.4.4 Grouping constraints:
1065  // trap, pause, barrier, icinva, isync, and syncht are solo instructions.
1066  // They must not be grouped with other instructions in a packet.
1067  if (isSchedBarrier(MI))
1068  return true;
1069 
1070  if (HII->isSolo(MI))
1071  return true;
1072 
1073  if (MI.getOpcode() == Hexagon::A2_nop)
1074  return true;
1075 
1076  return false;
1077 }
1078 
1079 // Quick check if instructions MI and MJ cannot coexist in the same packet.
1080 // Limit the tests to be "one-way", e.g. "if MI->isBranch and MJ->isInlineAsm",
1081 // but not the symmetric case: "if MJ->isBranch and MI->isInlineAsm".
1082 // For full test call this function twice:
1083 // cannotCoexistAsymm(MI, MJ) || cannotCoexistAsymm(MJ, MI)
1084 // Doing the test only one way saves the amount of code in this function,
1085 // since every test would need to be repeated with the MI and MJ reversed.
1086 static bool cannotCoexistAsymm(const MachineInstr &MI, const MachineInstr &MJ,
1087  const HexagonInstrInfo &HII) {
1088  const MachineFunction *MF = MI.getParent()->getParent();
1090  HII.isHVXMemWithAIndirect(MI, MJ))
1091  return true;
1092 
1093  // An inline asm cannot be together with a branch, because we may not be
1094  // able to remove the asm out after packetizing (i.e. if the asm must be
1095  // moved past the bundle). Similarly, two asms cannot be together to avoid
1096  // complications when determining their relative order outside of a bundle.
1097  if (MI.isInlineAsm())
1098  return MJ.isInlineAsm() || MJ.isBranch() || MJ.isBarrier() ||
1099  MJ.isCall() || MJ.isTerminator();
1100 
1101  switch (MI.getOpcode()) {
1102  case (Hexagon::S2_storew_locked):
1103  case (Hexagon::S4_stored_locked):
1104  case (Hexagon::L2_loadw_locked):
1105  case (Hexagon::L4_loadd_locked):
1106  case (Hexagon::Y4_l2fetch): {
1107  // These instructions can only be grouped with ALU32 or non-floating-point
1108  // XTYPE instructions. Since there is no convenient way of identifying fp
1109  // XTYPE instructions, only allow grouping with ALU32 for now.
1110  unsigned TJ = HII.getType(MJ);
1111  if (TJ != HexagonII::TypeALU32_2op &&
1112  TJ != HexagonII::TypeALU32_3op &&
1114  return true;
1115  break;
1116  }
1117  default:
1118  break;
1119  }
1120 
1121  // "False" really means that the quick check failed to determine if
1122  // I and J cannot coexist.
1123  return false;
1124 }
1125 
1126 // Full, symmetric check.
1128  const MachineInstr &MJ) {
1129  return cannotCoexistAsymm(MI, MJ, *HII) || cannotCoexistAsymm(MJ, MI, *HII);
1130 }
1131 
1133  for (auto &B : MF) {
1134  MachineBasicBlock::iterator BundleIt;
1136  for (auto I = B.instr_begin(), E = B.instr_end(); I != E; I = NextI) {
1137  NextI = std::next(I);
1138  MachineInstr &MI = *I;
1139  if (MI.isBundle())
1140  BundleIt = I;
1141  if (!MI.isInsideBundle())
1142  continue;
1143 
1144  // Decide on where to insert the instruction that we are pulling out.
1145  // Debug instructions always go before the bundle, but the placement of
1146  // INLINE_ASM depends on potential dependencies. By default, try to
1147  // put it before the bundle, but if the asm writes to a register that
1148  // other instructions in the bundle read, then we need to place it
1149  // after the bundle (to preserve the bundle semantics).
1150  bool InsertBeforeBundle;
1151  if (MI.isInlineAsm())
1152  InsertBeforeBundle = !hasWriteToReadDep(MI, *BundleIt, HRI);
1153  else if (MI.isDebugValue())
1154  InsertBeforeBundle = true;
1155  else
1156  continue;
1157 
1158  BundleIt = moveInstrOut(MI, BundleIt, InsertBeforeBundle);
1159  }
1160  }
1161 }
1162 
1163 // Check if a given instruction is of class "system".
1164 static bool isSystemInstr(const MachineInstr &MI) {
1165  unsigned Opc = MI.getOpcode();
1166  switch (Opc) {
1167  case Hexagon::Y2_barrier:
1168  case Hexagon::Y2_dcfetchbo:
1169  return true;
1170  }
1171  return false;
1172 }
1173 
1175  const MachineInstr &J) {
1176  // The dependence graph may not include edges between dead definitions,
1177  // so without extra checks, we could end up packetizing two instruction
1178  // defining the same (dead) register.
1179  if (I.isCall() || J.isCall())
1180  return false;
1181  if (HII->isPredicated(I) || HII->isPredicated(J))
1182  return false;
1183 
1184  BitVector DeadDefs(Hexagon::NUM_TARGET_REGS);
1185  for (auto &MO : I.operands()) {
1186  if (!MO.isReg() || !MO.isDef() || !MO.isDead())
1187  continue;
1188  DeadDefs[MO.getReg()] = true;
1189  }
1190 
1191  for (auto &MO : J.operands()) {
1192  if (!MO.isReg() || !MO.isDef() || !MO.isDead())
1193  continue;
1194  unsigned R = MO.getReg();
1195  if (R != Hexagon::USR_OVF && DeadDefs[R])
1196  return true;
1197  }
1198  return false;
1199 }
1200 
1202  const MachineInstr &J) {
1203  // A save callee-save register function call can only be in a packet
1204  // with instructions that don't write to the callee-save registers.
1205  if ((HII->isSaveCalleeSavedRegsCall(I) &&
1206  doesModifyCalleeSavedReg(J, HRI)) ||
1207  (HII->isSaveCalleeSavedRegsCall(J) &&
1208  doesModifyCalleeSavedReg(I, HRI)))
1209  return true;
1210 
1211  // Two control flow instructions cannot go in the same packet.
1212  if (isControlFlow(I) && isControlFlow(J))
1213  return true;
1214 
1215  // \ref-manual (7.3.4) A loop setup packet in loopN or spNloop0 cannot
1216  // contain a speculative indirect jump,
1217  // a new-value compare jump or a dealloc_return.
1218  auto isBadForLoopN = [this] (const MachineInstr &MI) -> bool {
1219  if (MI.isCall() || HII->isDeallocRet(MI) || HII->isNewValueJump(MI))
1220  return true;
1221  if (HII->isPredicated(MI) && HII->isPredicatedNew(MI) && HII->isJumpR(MI))
1222  return true;
1223  return false;
1224  };
1225 
1226  if (HII->isLoopN(I) && isBadForLoopN(J))
1227  return true;
1228  if (HII->isLoopN(J) && isBadForLoopN(I))
1229  return true;
1230 
1231  // dealloc_return cannot appear in the same packet as a conditional or
1232  // unconditional jump.
1233  return HII->isDeallocRet(I) &&
1234  (J.isBranch() || J.isCall() || J.isBarrier());
1235 }
1236 
1238  const MachineInstr &J) {
1239  // Adding I to a packet that has J.
1240 
1241  // Regmasks are not reflected in the scheduling dependency graph, so
1242  // we need to check them manually. This code assumes that regmasks only
1243  // occur on calls, and the problematic case is when we add an instruction
1244  // defining a register R to a packet that has a call that clobbers R via
1245  // a regmask. Those cannot be packetized together, because the call will
1246  // be executed last. That's also a reson why it is ok to add a call
1247  // clobbering R to a packet that defines R.
1248 
1249  // Look for regmasks in J.
1250  for (const MachineOperand &OpJ : J.operands()) {
1251  if (!OpJ.isRegMask())
1252  continue;
1253  assert((J.isCall() || HII->isTailCall(J)) && "Regmask on a non-call");
1254  for (const MachineOperand &OpI : I.operands()) {
1255  if (OpI.isReg()) {
1256  if (OpJ.clobbersPhysReg(OpI.getReg()))
1257  return true;
1258  } else if (OpI.isRegMask()) {
1259  // Both are regmasks. Assume that they intersect.
1260  return true;
1261  }
1262  }
1263  }
1264  return false;
1265 }
1266 
1268  const MachineInstr &J) {
1269  bool SysI = isSystemInstr(I), SysJ = isSystemInstr(J);
1270  bool StoreI = I.mayStore(), StoreJ = J.mayStore();
1271  if ((SysI && StoreJ) || (SysJ && StoreI))
1272  return true;
1273 
1274  if (StoreI && StoreJ) {
1275  if (HII->isNewValueInst(J) || HII->isMemOp(J) || HII->isMemOp(I))
1276  return true;
1277  } else {
1278  // A memop cannot be in the same packet with another memop or a store.
1279  // Two stores can be together, but here I and J cannot both be stores.
1280  bool MopStI = HII->isMemOp(I) || StoreI;
1281  bool MopStJ = HII->isMemOp(J) || StoreJ;
1282  if (MopStI && MopStJ)
1283  return true;
1284  }
1285 
1286  return (StoreJ && HII->isDeallocRet(I)) || (StoreI && HII->isDeallocRet(J));
1287 }
1288 
1289 // SUI is the current instruction that is out side of the current packet.
1290 // SUJ is the current instruction inside the current packet against which that
1291 // SUI will be packetized.
1293  assert(SUI->getInstr() && SUJ->getInstr());
1294  MachineInstr &I = *SUI->getInstr();
1295  MachineInstr &J = *SUJ->getInstr();
1296 
1297  // Clear IgnoreDepMIs when Packet starts.
1298  if (CurrentPacketMIs.size() == 1)
1299  IgnoreDepMIs.clear();
1300 
1301  MachineBasicBlock::iterator II = I.getIterator();
1302 
1303  // Solo instructions cannot go in the packet.
1304  assert(!isSoloInstruction(I) && "Unexpected solo instr!");
1305 
1306  if (cannotCoexist(I, J))
1307  return false;
1308 
1309  Dependence = hasDeadDependence(I, J) || hasControlDependence(I, J);
1310  if (Dependence)
1311  return false;
1312 
1313  // Regmasks are not accounted for in the scheduling graph, so we need
1314  // to explicitly check for dependencies caused by them. They should only
1315  // appear on calls, so it's not too pessimistic to reject all regmask
1316  // dependencies.
1317  Dependence = hasRegMaskDependence(I, J);
1318  if (Dependence)
1319  return false;
1320 
1321  // V4 allows dual stores. It does not allow second store, if the first
1322  // store is not in SLOT0. New value store, new value jump, dealloc_return
1323  // and memop always take SLOT0. Arch spec 3.4.4.2.
1324  Dependence = hasV4SpecificDependence(I, J);
1325  if (Dependence)
1326  return false;
1327 
1328  // If an instruction feeds new value jump, glue it.
1329  MachineBasicBlock::iterator NextMII = I.getIterator();
1330  ++NextMII;
1331  if (NextMII != I.getParent()->end() && HII->isNewValueJump(*NextMII)) {
1332  MachineInstr &NextMI = *NextMII;
1333 
1334  bool secondRegMatch = false;
1335  const MachineOperand &NOp0 = NextMI.getOperand(0);
1336  const MachineOperand &NOp1 = NextMI.getOperand(1);
1337 
1338  if (NOp1.isReg() && I.getOperand(0).getReg() == NOp1.getReg())
1339  secondRegMatch = true;
1340 
1341  for (MachineInstr *PI : CurrentPacketMIs) {
1342  // NVJ can not be part of the dual jump - Arch Spec: section 7.8.
1343  if (PI->isCall()) {
1344  Dependence = true;
1345  break;
1346  }
1347  // Validate:
1348  // 1. Packet does not have a store in it.
1349  // 2. If the first operand of the nvj is newified, and the second
1350  // operand is also a reg, it (second reg) is not defined in
1351  // the same packet.
1352  // 3. If the second operand of the nvj is newified, (which means
1353  // first operand is also a reg), first reg is not defined in
1354  // the same packet.
1355  if (PI->getOpcode() == Hexagon::S2_allocframe || PI->mayStore() ||
1356  HII->isLoopN(*PI)) {
1357  Dependence = true;
1358  break;
1359  }
1360  // Check #2/#3.
1361  const MachineOperand &OpR = secondRegMatch ? NOp0 : NOp1;
1362  if (OpR.isReg() && PI->modifiesRegister(OpR.getReg(), HRI)) {
1363  Dependence = true;
1364  break;
1365  }
1366  }
1367 
1368  GlueToNewValueJump = true;
1369  if (Dependence)
1370  return false;
1371  }
1372 
1373  // There no dependency between a prolog instruction and its successor.
1374  if (!SUJ->isSucc(SUI))
1375  return true;
1376 
1377  for (unsigned i = 0; i < SUJ->Succs.size(); ++i) {
1378  if (FoundSequentialDependence)
1379  break;
1380 
1381  if (SUJ->Succs[i].getSUnit() != SUI)
1382  continue;
1383 
1384  SDep::Kind DepType = SUJ->Succs[i].getKind();
1385  // For direct calls:
1386  // Ignore register dependences for call instructions for packetization
1387  // purposes except for those due to r31 and predicate registers.
1388  //
1389  // For indirect calls:
1390  // Same as direct calls + check for true dependences to the register
1391  // used in the indirect call.
1392  //
1393  // We completely ignore Order dependences for call instructions.
1394  //
1395  // For returns:
1396  // Ignore register dependences for return instructions like jumpr,
1397  // dealloc return unless we have dependencies on the explicit uses
1398  // of the registers used by jumpr (like r31) or dealloc return
1399  // (like r29 or r30).
1400  unsigned DepReg = 0;
1401  const TargetRegisterClass *RC = nullptr;
1402  if (DepType == SDep::Data) {
1403  DepReg = SUJ->Succs[i].getReg();
1404  RC = HRI->getMinimalPhysRegClass(DepReg);
1405  }
1406 
1407  if (I.isCall() || HII->isJumpR(I) || I.isReturn() || HII->isTailCall(I)) {
1408  if (!isRegDependence(DepType))
1409  continue;
1410  if (!isCallDependent(I, DepType, SUJ->Succs[i].getReg()))
1411  continue;
1412  }
1413 
1414  if (DepType == SDep::Data) {
1415  if (canPromoteToDotCur(J, SUJ, DepReg, II, RC))
1416  if (promoteToDotCur(J, DepType, II, RC))
1417  continue;
1418  }
1419 
1420  // Data dpendence ok if we have load.cur.
1421  if (DepType == SDep::Data && HII->isDotCurInst(J)) {
1422  if (HII->isHVXVec(I))
1423  continue;
1424  }
1425 
1426  // For instructions that can be promoted to dot-new, try to promote.
1427  if (DepType == SDep::Data) {
1428  if (canPromoteToDotNew(I, SUJ, DepReg, II, RC)) {
1429  if (promoteToDotNew(I, DepType, II, RC)) {
1430  PromotedToDotNew = true;
1431  if (cannotCoexist(I, J))
1432  FoundSequentialDependence = true;
1433  continue;
1434  }
1435  }
1436  if (HII->isNewValueJump(I))
1437  continue;
1438  }
1439 
1440  // For predicated instructions, if the predicates are complements then
1441  // there can be no dependence.
1442  if (HII->isPredicated(I) && HII->isPredicated(J) &&
1443  arePredicatesComplements(I, J)) {
1444  // Not always safe to do this translation.
1445  // DAG Builder attempts to reduce dependence edges using transitive
1446  // nature of dependencies. Here is an example:
1447  //
1448  // r0 = tfr_pt ... (1)
1449  // r0 = tfr_pf ... (2)
1450  // r0 = tfr_pt ... (3)
1451  //
1452  // There will be an output dependence between (1)->(2) and (2)->(3).
1453  // However, there is no dependence edge between (1)->(3). This results
1454  // in all 3 instructions going in the same packet. We ignore dependce
1455  // only once to avoid this situation.
1456  auto Itr = find(IgnoreDepMIs, &J);
1457  if (Itr != IgnoreDepMIs.end()) {
1458  Dependence = true;
1459  return false;
1460  }
1461  IgnoreDepMIs.push_back(&I);
1462  continue;
1463  }
1464 
1465  // Ignore Order dependences between unconditional direct branches
1466  // and non-control-flow instructions.
1467  if (isDirectJump(I) && !J.isBranch() && !J.isCall() &&
1468  DepType == SDep::Order)
1469  continue;
1470 
1471  // Ignore all dependences for jumps except for true and output
1472  // dependences.
1473  if (I.isConditionalBranch() && DepType != SDep::Data &&
1474  DepType != SDep::Output)
1475  continue;
1476 
1477  if (DepType == SDep::Output) {
1478  FoundSequentialDependence = true;
1479  break;
1480  }
1481 
1482  // For Order dependences:
1483  // 1. On V4 or later, volatile loads/stores can be packetized together,
1484  // unless other rules prevent is.
1485  // 2. Store followed by a load is not allowed.
1486  // 3. Store followed by a store is only valid on V4 or later.
1487  // 4. Load followed by any memory operation is allowed.
1488  if (DepType == SDep::Order) {
1489  if (!PacketizeVolatiles) {
1490  bool OrdRefs = I.hasOrderedMemoryRef() || J.hasOrderedMemoryRef();
1491  if (OrdRefs) {
1492  FoundSequentialDependence = true;
1493  break;
1494  }
1495  }
1496  // J is first, I is second.
1497  bool LoadJ = J.mayLoad(), StoreJ = J.mayStore();
1498  bool LoadI = I.mayLoad(), StoreI = I.mayStore();
1499  if (StoreJ) {
1500  // Two stores are only allowed on V4+. Load following store is never
1501  // allowed.
1502  if (LoadI && alias(J, I)) {
1503  FoundSequentialDependence = true;
1504  break;
1505  }
1506  } else if (!LoadJ || (!LoadI && !StoreI)) {
1507  // If J is neither load nor store, assume a dependency.
1508  // If J is a load, but I is neither, also assume a dependency.
1509  FoundSequentialDependence = true;
1510  break;
1511  }
1512  // Store followed by store: not OK on V2.
1513  // Store followed by load: not OK on all.
1514  // Load followed by store: OK on all.
1515  // Load followed by load: OK on all.
1516  continue;
1517  }
1518 
1519  // For V4, special case ALLOCFRAME. Even though there is dependency
1520  // between ALLOCFRAME and subsequent store, allow it to be packetized
1521  // in a same packet. This implies that the store is using the caller's
1522  // SP. Hence, offset needs to be updated accordingly.
1523  if (DepType == SDep::Data && J.getOpcode() == Hexagon::S2_allocframe) {
1524  unsigned Opc = I.getOpcode();
1525  switch (Opc) {
1526  case Hexagon::S2_storerd_io:
1527  case Hexagon::S2_storeri_io:
1528  case Hexagon::S2_storerh_io:
1529  case Hexagon::S2_storerb_io:
1530  if (I.getOperand(0).getReg() == HRI->getStackRegister()) {
1531  // Since this store is to be glued with allocframe in the same
1532  // packet, it will use SP of the previous stack frame, i.e.
1533  // caller's SP. Therefore, we need to recalculate offset
1534  // according to this change.
1535  GlueAllocframeStore = useCallersSP(I);
1536  if (GlueAllocframeStore)
1537  continue;
1538  }
1539  default:
1540  break;
1541  }
1542  }
1543 
1544  // There are certain anti-dependencies that cannot be ignored.
1545  // Specifically:
1546  // J2_call ... %R0<imp-def> ; SUJ
1547  // R0 = ... ; SUI
1548  // Those cannot be packetized together, since the call will observe
1549  // the effect of the assignment to R0.
1550  if ((DepType == SDep::Anti || DepType == SDep::Output) && J.isCall()) {
1551  // Check if I defines any volatile register. We should also check
1552  // registers that the call may read, but these happen to be a
1553  // subset of the volatile register set.
1554  for (const MachineOperand &Op : I.operands()) {
1555  if (Op.isReg() && Op.isDef()) {
1556  unsigned R = Op.getReg();
1557  if (!J.readsRegister(R, HRI) && !J.modifiesRegister(R, HRI))
1558  continue;
1559  } else if (!Op.isRegMask()) {
1560  // If I has a regmask assume dependency.
1561  continue;
1562  }
1563  FoundSequentialDependence = true;
1564  break;
1565  }
1566  }
1567 
1568  // Skip over remaining anti-dependences. Two instructions that are
1569  // anti-dependent can share a packet, since in most such cases all
1570  // operands are read before any modifications take place.
1571  // The exceptions are branch and call instructions, since they are
1572  // executed after all other instructions have completed (at least
1573  // conceptually).
1574  if (DepType != SDep::Anti) {
1575  FoundSequentialDependence = true;
1576  break;
1577  }
1578  }
1579 
1580  if (FoundSequentialDependence) {
1581  Dependence = true;
1582  return false;
1583  }
1584 
1585  return true;
1586 }
1587 
1589  assert(SUI->getInstr() && SUJ->getInstr());
1590  MachineInstr &I = *SUI->getInstr();
1591  MachineInstr &J = *SUJ->getInstr();
1592 
1593  bool Coexist = !cannotCoexist(I, J);
1594 
1595  if (Coexist && !Dependence)
1596  return true;
1597 
1598  // Check if the instruction was promoted to a dot-new. If so, demote it
1599  // back into a dot-old.
1600  if (PromotedToDotNew)
1601  demoteToDotOld(I);
1602 
1603  cleanUpDotCur();
1604  // Check if the instruction (must be a store) was glued with an allocframe
1605  // instruction. If so, restore its offset to its original value, i.e. use
1606  // current SP instead of caller's SP.
1607  if (GlueAllocframeStore) {
1608  useCalleesSP(I);
1609  GlueAllocframeStore = false;
1610  }
1611 
1612  if (ChangedOffset != INT64_MAX)
1613  undoChangedOffset(I);
1614 
1615  if (GlueToNewValueJump) {
1616  // Putting I and J together would prevent the new-value jump from being
1617  // packetized with the producer. In that case I and J must be separated.
1618  GlueToNewValueJump = false;
1619  return false;
1620  }
1621 
1622  if (ChangedOffset == INT64_MAX && updateOffset(SUI, SUJ)) {
1623  FoundSequentialDependence = false;
1624  Dependence = false;
1625  return true;
1626  }
1627 
1628  return false;
1629 }
1630 
1634  MachineBasicBlock *MBB = MI.getParent();
1635 
1636  if (CurrentPacketMIs.empty())
1637  PacketStalls = false;
1638  PacketStalls |= producesStall(MI);
1639 
1640  if (MI.isImplicitDef())
1641  return MII;
1642  assert(ResourceTracker->canReserveResources(MI));
1643 
1644  bool ExtMI = HII->isExtended(MI) || HII->isConstExtended(MI);
1645  bool Good = true;
1646 
1647  if (GlueToNewValueJump) {
1648  MachineInstr &NvjMI = *++MII;
1649  // We need to put both instructions in the same packet: MI and NvjMI.
1650  // Either of them can require a constant extender. Try to add both to
1651  // the current packet, and if that fails, end the packet and start a
1652  // new one.
1653  ResourceTracker->reserveResources(MI);
1654  if (ExtMI)
1655  Good = tryAllocateResourcesForConstExt(true);
1656 
1657  bool ExtNvjMI = HII->isExtended(NvjMI) || HII->isConstExtended(NvjMI);
1658  if (Good) {
1659  if (ResourceTracker->canReserveResources(NvjMI))
1660  ResourceTracker->reserveResources(NvjMI);
1661  else
1662  Good = false;
1663  }
1664  if (Good && ExtNvjMI)
1665  Good = tryAllocateResourcesForConstExt(true);
1666 
1667  if (!Good) {
1668  endPacket(MBB, MI);
1669  assert(ResourceTracker->canReserveResources(MI));
1670  ResourceTracker->reserveResources(MI);
1671  if (ExtMI) {
1672  assert(canReserveResourcesForConstExt());
1673  tryAllocateResourcesForConstExt(true);
1674  }
1675  assert(ResourceTracker->canReserveResources(NvjMI));
1676  ResourceTracker->reserveResources(NvjMI);
1677  if (ExtNvjMI) {
1678  assert(canReserveResourcesForConstExt());
1679  reserveResourcesForConstExt();
1680  }
1681  }
1682  CurrentPacketMIs.push_back(&MI);
1683  CurrentPacketMIs.push_back(&NvjMI);
1684  return MII;
1685  }
1686 
1687  ResourceTracker->reserveResources(MI);
1688  if (ExtMI && !tryAllocateResourcesForConstExt(true)) {
1689  endPacket(MBB, MI);
1690  if (PromotedToDotNew)
1691  demoteToDotOld(MI);
1692  if (GlueAllocframeStore) {
1693  useCalleesSP(MI);
1694  GlueAllocframeStore = false;
1695  }
1696  ResourceTracker->reserveResources(MI);
1697  reserveResourcesForConstExt();
1698  }
1699 
1700  CurrentPacketMIs.push_back(&MI);
1701  return MII;
1702 }
1703 
1706  OldPacketMIs = CurrentPacketMIs;
1708 }
1709 
1711  return !producesStall(MI);
1712 }
1713 
1714 // V60 forward scheduling.
1716  // If the packet already stalls, then ignore the stall from a subsequent
1717  // instruction in the same packet.
1718  if (PacketStalls)
1719  return false;
1720 
1721  // Check whether the previous packet is in a different loop. If this is the
1722  // case, there is little point in trying to avoid a stall because that would
1723  // favor the rare case (loop entry) over the common case (loop iteration).
1724  //
1725  // TODO: We should really be able to check all the incoming edges if this is
1726  // the first packet in a basic block, so we can avoid stalls from the loop
1727  // backedge.
1728  if (!OldPacketMIs.empty()) {
1729  auto *OldBB = OldPacketMIs.front()->getParent();
1730  auto *ThisBB = I.getParent();
1731  if (MLI->getLoopFor(OldBB) != MLI->getLoopFor(ThisBB))
1732  return false;
1733  }
1734 
1735  SUnit *SUI = MIToSUnit[const_cast<MachineInstr *>(&I)];
1736 
1737  // Check if the latency is 0 between this instruction and any instruction
1738  // in the current packet. If so, we disregard any potential stalls due to
1739  // the instructions in the previous packet. Most of the instruction pairs
1740  // that can go together in the same packet have 0 latency between them.
1741  // Only exceptions are newValueJumps as they're generated much later and
1742  // the latencies can't be changed at that point. Another is .cur
1743  // instructions if its consumer has a 0 latency successor (such as .new).
1744  // In this case, the latency between .cur and the consumer stays non-zero
1745  // even though we can have both .cur and .new in the same packet. Changing
1746  // the latency to 0 is not an option as it causes software pipeliner to
1747  // not pipeline in some cases.
1748 
1749  // For Example:
1750  // {
1751  // I1: v6.cur = vmem(r0++#1)
1752  // I2: v7 = valign(v6,v4,r2)
1753  // I3: vmem(r5++#1) = v7.new
1754  // }
1755  // Here I2 and I3 has 0 cycle latency, but I1 and I2 has 2.
1756 
1757  for (auto J : CurrentPacketMIs) {
1758  SUnit *SUJ = MIToSUnit[J];
1759  for (auto &Pred : SUI->Preds)
1760  if (Pred.getSUnit() == SUJ &&
1761  (Pred.getLatency() == 0 || HII->isNewValueJump(I) ||
1762  HII->isToBeScheduledASAP(*J, I)))
1763  return false;
1764  }
1765 
1766  // Check if the latency is greater than one between this instruction and any
1767  // instruction in the previous packet.
1768  for (auto J : OldPacketMIs) {
1769  SUnit *SUJ = MIToSUnit[J];
1770  for (auto &Pred : SUI->Preds)
1771  if (Pred.getSUnit() == SUJ && Pred.getLatency() > 1)
1772  return true;
1773  }
1774 
1775  // Check if the latency is greater than one between this instruction and any
1776  // instruction in the previous packet.
1777  for (auto J : OldPacketMIs) {
1778  SUnit *SUJ = MIToSUnit[J];
1779  for (auto &Pred : SUI->Preds)
1780  if (Pred.getSUnit() == SUJ && Pred.getLatency() > 1)
1781  return true;
1782  }
1783 
1784  return false;
1785 }
1786 
1787 //===----------------------------------------------------------------------===//
1788 // Public Constructor Functions
1789 //===----------------------------------------------------------------------===//
1790 
1792  return new HexagonPacketizer();
1793 }
bool canPromoteToNewValueStore(const MachineInstr &MI, const MachineInstr &PacketMI, unsigned DepReg)
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static bool hasWriteToReadDep(const MachineInstr &FirstI, const MachineInstr &SecondI, const TargetRegisterInfo *TRI)
bool modifiesRegister(unsigned Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr modifies (fully define or partially define) the specified register...
Definition: MachineInstr.h:965
bool isCall(QueryType Type=AnyInBundle) const
Definition: MachineInstr.h:458
instr_iterator instr_end()
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
bool canPromoteToDotNew(const MachineInstr &MI, const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII, const TargetRegisterClass *RC)
static bool cannotCoexistAsymm(const MachineInstr &MI, const MachineInstr &MJ, const HexagonInstrInfo &HII)
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
bool isCFIInstruction() const
Definition: MachineInstr.h:809
bool isBundledWithPred() const
Return true if this instruction is part of a bundle, and it is not the first instruction in the bundl...
Definition: MachineInstr.h:247
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
unsigned getReg() const
getReg - Returns the register number.
static bool isImplicitDependency(const MachineInstr &I, bool CheckDef, unsigned DepReg)
bool isInlineAsm() const
Definition: MachineInstr.h:832
bool demoteToDotOld(MachineInstr &MI)
static cl::opt< bool > PacketizeVolatiles("hexagon-packetize-volatiles", cl::ZeroOrMore, cl::Hidden, cl::init(true), cl::desc("Allow non-solo packetization of volatile memory references"))
void undoChangedOffset(MachineInstr &MI)
Undo the changed offset.
static cl::opt< bool > DisablePacketizer("disable-packetizer", cl::Hidden, cl::ZeroOrMore, cl::init(false), cl::desc("Disable Hexagon packetizer pass"))
INITIALIZE_PASS_BEGIN(HexagonPacketizer, "hexagon-packetizer", "Hexagon Packetizer", false, false) INITIALIZE_PASS_END(HexagonPacketizer
A debug info location.
Definition: DebugLoc.h:34
Kind
These are the different kinds of scheduling dependencies.
Definition: ScheduleDAG.h:53
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:332
MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, const DebugLoc &DL, bool NoImp=false)
CreateMachineInstr - Allocate a new MachineInstr.
SmallVector< SDep, 4 > Preds
All sunit predecessors.
Definition: ScheduleDAG.h:261
A register anti-dependence (aka WAR).
Definition: ScheduleDAG.h:55
bool restrictingDepExistInPacket(MachineInstr &, unsigned)
bool canPromoteToDotCur(const MachineInstr &MI, const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII, const TargetRegisterClass *RC)
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:51
static cl::opt< bool > EnableGenAllInsnClass("enable-gen-insn", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Generate all instruction with TC"))
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
unsigned getNumOperands() const
Access to explicit operands of the instruction.
Definition: MachineInstr.h:293
bool isBundledWithSucc() const
Return true if this instruction is part of a bundle, and it is not the last instruction in the bundle...
Definition: MachineInstr.h:251
bool shouldAddToPacket(const MachineInstr &MI) override
Regular data dependence (aka true-dependence).
Definition: ScheduleDAG.h:54
virtual void endPacket(MachineBasicBlock *MBB, MachineBasicBlock::iterator MI)
Reg
All possible values of the reg field in the ModR/M byte.
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
Definition: MachineInstr.h:474
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:290
void endPacket(MachineBasicBlock *MBB, MachineBasicBlock::iterator MI) override
bool arePredicatesComplements(MachineInstr &MI1, MachineInstr &MI2)
static bool doesModifyCalleeSavedReg(const MachineInstr &MI, const TargetRegisterInfo *TRI)
Returns true if the instruction modifies a callee-saved register.
void unbundleFromPred()
Break bundle above this instruction.
A register output-dependence (aka WAW).
Definition: ScheduleDAG.h:56
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:287
static bool isControlFlow(const MachineInstr &MI)
bool isBundle() const
Definition: MachineInstr.h:853
void initializeHexagonPacketizerPass(PassRegistry &)
bool isInsideBundle() const
Return true if MI is in a bundle (but not the first MI in a bundle).
Definition: MachineInstr.h:235
virtual const MCPhysReg * getCalleeSavedRegs(const MachineFunction *MF) const =0
Return a null-terminated list of all of the callee-saved registers on this target.
hexagon Hexagon Packetizer
bool isBranch(QueryType Type=AnyInBundle) const
Returns true if this is a conditional, unconditional, or indirect branch.
Definition: MachineInstr.h:482
static MachineBasicBlock::iterator moveInstrOut(MachineInstr &MI, MachineBasicBlock::iterator BundleIt, bool Before)
static unsigned getPredicatedRegister(MachineInstr &MI, const HexagonInstrInfo *QII)
Gets the predicate register of a predicated instruction.
static const MachineOperand & getAbsSetOperand(const MachineInstr &MI)
bool isSoloInstruction(const MachineInstr &MI) override
unsigned getSchedClass() const
Return the scheduling class for this instruction.
Definition: MCInstrDesc.h:565
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:639
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:406
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:378
static bool isSystemInstr(const MachineInstr &MI)
FunctionPass * createHexagonPacketizer()
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:187
Represent the analysis usage information of a pass.
void clearFlag(MIFlag Flag)
clearFlag - Clear a MI flag.
Definition: MachineInstr.h:196
bool isEHLabel() const
Definition: MachineInstr.h:798
static const unsigned End
void setImm(int64_t immVal)
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:285
self_iterator getIterator()
Definition: ilist_node.h:82
static bool isRegDependence(const SDep::Kind DepType)
bool tryAllocateResourcesForConstExt(bool Reserve)
void DeleteMachineInstr(MachineInstr *MI)
DeleteMachineInstr - Delete the given MachineInstr.
bool isImplicitDef() const
Definition: MachineInstr.h:831
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Any other ordering dependency.
Definition: ScheduleDAG.h:57
auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:834
bool isPostIncrement(const MachineInstr &MI) const override
Return true for post-incremented instructions.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
bool isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) override
bool isPredicated(const MachineInstr &MI) const override
Returns true if the instruction is already predicated.
static const MachineOperand & getPostIncrementOperand(const MachineInstr &MI, const HexagonInstrInfo *HII)
#define HEXAGON_LRFP_SIZE
Definition: Hexagon.h:27
Iterator for intrusive lists based on ilist_node.
void setDesc(const MCInstrDesc &tid)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one...
bool isDebugValue() const
Definition: MachineInstr.h:816
MachineOperand class - Representation of each machine instruction operand.
uint64_t getType(const MachineInstr &MI) const
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
static cl::opt< bool > DisableVecDblNVStores("disable-vecdbl-nv-stores", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Disable vector double new-value-stores"))
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:285
int64_t getImm() const
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
MachineBasicBlock::iterator addToPacket(MachineInstr &MI) override
bool readsRegister(unsigned Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr reads the specified register.
Definition: MachineInstr.h:927
bool mayStore() const
Return true if this instruction could possibly modify memory.
Definition: MCInstrDesc.h:393
static const MachineOperand & getStoreValueOperand(const MachineInstr &MI)
static bool isLoadAbsSet(const MachineInstr &MI)
static PredicateKind getPredicateSense(const MachineInstr &MI, const HexagonInstrInfo *HII)
Returns true if an instruction is predicated on p0 and false if it&#39;s predicated on !p0...
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:139
MachineFunctionProperties & set(Property P)
Representation of each machine instruction.
Definition: MachineInstr.h:59
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
static bool isSchedBarrier(const MachineInstr &MI)
bool promoteToDotCur(MachineInstr &MI, SDep::Kind DepType, MachineBasicBlock::iterator &MII, const TargetRegisterClass *RC)
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB &#39;Other&#39; at the position From, and insert it into this MBB right before &#39;...
bool isCallDependent(const MachineInstr &MI, SDep::Kind DepType, unsigned DepReg)
bool ignorePseudoInstruction(const MachineInstr &MI, const MachineBasicBlock *MBB) override
bool isNewifiable(const MachineInstr &MI, const TargetRegisterClass *NewRC)
bool updateOffset(SUnit *SUI, SUnit *SUJ)
Return true if we can update the offset in MI so that MI and MJ can be packetized together...
cl::opt< bool > ScheduleInlineAsm
#define I(x, y, z)
Definition: MD5.cpp:58
bool isCall() const
Return true if the instruction is a call.
Definition: MCInstrDesc.h:251
bool isPredicatedTrue(const MachineInstr &MI) const
bool hasControlDependence(const MachineInstr &I, const MachineInstr &J)
DFAPacketizer * getResourceTracker()
bool useCallersSP(MachineInstr &MI)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
hexagon packetizer
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
Definition: MachineInstr.h:626
bool promoteToDotNew(MachineInstr &MI, SDep::Kind DepType, MachineBasicBlock::iterator &MII, const TargetRegisterClass *RC)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool hasDeadDependence(const MachineInstr &I, const MachineInstr &J)
bool isHVXMemWithAIndirect(const MachineInstr &I, const MachineInstr &J) const
bool isTerminator() const
Returns true if this instruction part of the terminator for a basic block.
Definition: MCInstrDesc.h:264
aarch64 promote const
bool cannotCoexist(const MachineInstr &MI, const MachineInstr &MJ)
bool isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) override
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
SmallVector< SDep, 4 > Succs
All sunit successors.
Definition: ScheduleDAG.h:262
#define DEBUG(X)
Definition: Debug.h:118
void useCalleesSP(MachineInstr &MI)
bool hasRegMaskDependence(const MachineInstr &I, const MachineInstr &J)
IRTranslator LLVM IR MI
bool isBarrier(QueryType Type=AnyInBundle) const
Returns true if the specified instruction stops control flow from executing the instruction immediate...
Definition: MachineInstr.h:465
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
Definition: PassRegistry.h:39
void unpacketizeSoloInstrs(MachineFunction &MF)
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
Dependence - This class represents a dependence between two memory memory references in a function...
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:295
bool canPromoteToNewValue(const MachineInstr &MI, const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII)
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects...
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
bool isSucc(const SUnit *N) const
Tests if node N is a successor of this node.
Definition: ScheduleDAG.h:444
#define OP(n)
Definition: regex2.h:73
void PacketizeMIs(MachineBasicBlock *MBB, MachineBasicBlock::iterator BeginItr, MachineBasicBlock::iterator EndItr)
static bool isDirectJump(const MachineInstr &MI)
bool producesStall(const MachineInstr &MI)
Properties which a MachineFunction may have at a given point in time.
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:247
bool hasV4SpecificDependence(const MachineInstr &I, const MachineInstr &J)