LLVM  3.7.0
ScheduleDAGRRList.cpp
Go to the documentation of this file.
1 //===----- ScheduleDAGRRList.cpp - Reg pressure reduction list scheduler --===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements bottom-up and top-down register pressure reduction list
11 // schedulers, using standard algorithms. The basic approach uses a priority
12 // queue of available nodes to schedule. One at a time, nodes are taken from
13 // the priority queue (thus in priority order), checked for legality to
14 // schedule, and emitted if legal.
15 //
16 //===----------------------------------------------------------------------===//
17 
19 #include "ScheduleDAGSDNodes.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
26 #include "llvm/IR/DataLayout.h"
27 #include "llvm/IR/InlineAsm.h"
28 #include "llvm/Support/Debug.h"
35 #include <climits>
36 using namespace llvm;
37 
38 #define DEBUG_TYPE "pre-RA-sched"
39 
40 STATISTIC(NumBacktracks, "Number of times scheduler backtracked");
41 STATISTIC(NumUnfolds, "Number of nodes unfolded");
42 STATISTIC(NumDups, "Number of duplicated nodes");
43 STATISTIC(NumPRCopies, "Number of physical register copies");
44 
45 static RegisterScheduler
46  burrListDAGScheduler("list-burr",
47  "Bottom-up register reduction list scheduling",
49 static RegisterScheduler
50  sourceListDAGScheduler("source",
51  "Similar to list-burr but schedules in source "
52  "order when possible",
54 
55 static RegisterScheduler
56  hybridListDAGScheduler("list-hybrid",
57  "Bottom-up register pressure aware list scheduling "
58  "which tries to balance latency and register pressure",
60 
61 static RegisterScheduler
62  ILPListDAGScheduler("list-ilp",
63  "Bottom-up register pressure aware list scheduling "
64  "which tries to balance ILP and register pressure",
66 
68  "disable-sched-cycles", cl::Hidden, cl::init(false),
69  cl::desc("Disable cycle-level precision during preRA scheduling"));
70 
71 // Temporary sched=list-ilp flags until the heuristics are robust.
72 // Some options are also available under sched=list-hybrid.
74  "disable-sched-reg-pressure", cl::Hidden, cl::init(false),
75  cl::desc("Disable regpressure priority in sched=list-ilp"));
77  "disable-sched-live-uses", cl::Hidden, cl::init(true),
78  cl::desc("Disable live use priority in sched=list-ilp"));
80  "disable-sched-vrcycle", cl::Hidden, cl::init(false),
81  cl::desc("Disable virtual register cycle interference checks"));
83  "disable-sched-physreg-join", cl::Hidden, cl::init(false),
84  cl::desc("Disable physreg def-use affinity"));
86  "disable-sched-stalls", cl::Hidden, cl::init(true),
87  cl::desc("Disable no-stall priority in sched=list-ilp"));
89  "disable-sched-critical-path", cl::Hidden, cl::init(false),
90  cl::desc("Disable critical path priority in sched=list-ilp"));
92  "disable-sched-height", cl::Hidden, cl::init(false),
93  cl::desc("Disable scheduled-height priority in sched=list-ilp"));
95  "disable-2addr-hack", cl::Hidden, cl::init(true),
96  cl::desc("Disable scheduler's two-address hack"));
97 
99  "max-sched-reorder", cl::Hidden, cl::init(6),
100  cl::desc("Number of instructions to allow ahead of the critical path "
101  "in sched=list-ilp"));
102 
104  "sched-avg-ipc", cl::Hidden, cl::init(1),
105  cl::desc("Average inst/cycle whan no target itinerary exists."));
106 
107 namespace {
108 //===----------------------------------------------------------------------===//
109 /// ScheduleDAGRRList - The actual register reduction list scheduler
110 /// implementation. This supports both top-down and bottom-up scheduling.
111 ///
112 class ScheduleDAGRRList : public ScheduleDAGSDNodes {
113 private:
114  /// NeedLatency - True if the scheduler will make use of latency information.
115  ///
116  bool NeedLatency;
117 
118  /// AvailableQueue - The priority queue to use for the available SUnits.
119  SchedulingPriorityQueue *AvailableQueue;
120 
121  /// PendingQueue - This contains all of the instructions whose operands have
122  /// been issued, but their results are not ready yet (due to the latency of
123  /// the operation). Once the operands becomes available, the instruction is
124  /// added to the AvailableQueue.
125  std::vector<SUnit*> PendingQueue;
126 
127  /// HazardRec - The hazard recognizer to use.
128  ScheduleHazardRecognizer *HazardRec;
129 
130  /// CurCycle - The current scheduler state corresponds to this cycle.
131  unsigned CurCycle;
132 
133  /// MinAvailableCycle - Cycle of the soonest available instruction.
134  unsigned MinAvailableCycle;
135 
136  /// IssueCount - Count instructions issued in this cycle
137  /// Currently valid only for bottom-up scheduling.
138  unsigned IssueCount;
139 
140  /// LiveRegDefs - A set of physical registers and their definition
141  /// that are "live". These nodes must be scheduled before any other nodes that
142  /// modifies the registers can be scheduled.
143  unsigned NumLiveRegs;
144  std::vector<SUnit*> LiveRegDefs;
145  std::vector<SUnit*> LiveRegGens;
146 
147  // Collect interferences between physical register use/defs.
148  // Each interference is an SUnit and set of physical registers.
149  SmallVector<SUnit*, 4> Interferences;
150  typedef DenseMap<SUnit*, SmallVector<unsigned, 4> > LRegsMapT;
151  LRegsMapT LRegsMap;
152 
153  /// Topo - A topological ordering for SUnits which permits fast IsReachable
154  /// and similar queries.
156 
157  // Hack to keep track of the inverse of FindCallSeqStart without more crazy
158  // DAG crawling.
159  DenseMap<SUnit*, SUnit*> CallSeqEndForStart;
160 
161 public:
162  ScheduleDAGRRList(MachineFunction &mf, bool needlatency,
163  SchedulingPriorityQueue *availqueue,
164  CodeGenOpt::Level OptLevel)
165  : ScheduleDAGSDNodes(mf),
166  NeedLatency(needlatency), AvailableQueue(availqueue), CurCycle(0),
167  Topo(SUnits, nullptr) {
168 
169  const TargetSubtargetInfo &STI = mf.getSubtarget();
170  if (DisableSchedCycles || !NeedLatency)
171  HazardRec = new ScheduleHazardRecognizer();
172  else
173  HazardRec = STI.getInstrInfo()->CreateTargetHazardRecognizer(&STI, this);
174  }
175 
176  ~ScheduleDAGRRList() override {
177  delete HazardRec;
178  delete AvailableQueue;
179  }
180 
181  void Schedule() override;
182 
183  ScheduleHazardRecognizer *getHazardRec() { return HazardRec; }
184 
185  /// IsReachable - Checks if SU is reachable from TargetSU.
186  bool IsReachable(const SUnit *SU, const SUnit *TargetSU) {
187  return Topo.IsReachable(SU, TargetSU);
188  }
189 
190  /// WillCreateCycle - Returns true if adding an edge from SU to TargetSU will
191  /// create a cycle.
192  bool WillCreateCycle(SUnit *SU, SUnit *TargetSU) {
193  return Topo.WillCreateCycle(SU, TargetSU);
194  }
195 
196  /// AddPred - adds a predecessor edge to SUnit SU.
197  /// This returns true if this is a new predecessor.
198  /// Updates the topological ordering if required.
199  void AddPred(SUnit *SU, const SDep &D) {
200  Topo.AddPred(SU, D.getSUnit());
201  SU->addPred(D);
202  }
203 
204  /// RemovePred - removes a predecessor edge from SUnit SU.
205  /// This returns true if an edge was removed.
206  /// Updates the topological ordering if required.
207  void RemovePred(SUnit *SU, const SDep &D) {
208  Topo.RemovePred(SU, D.getSUnit());
209  SU->removePred(D);
210  }
211 
212 private:
213  bool isReady(SUnit *SU) {
214  return DisableSchedCycles || !AvailableQueue->hasReadyFilter() ||
215  AvailableQueue->isReady(SU);
216  }
217 
218  void ReleasePred(SUnit *SU, const SDep *PredEdge);
219  void ReleasePredecessors(SUnit *SU);
220  void ReleasePending();
221  void AdvanceToCycle(unsigned NextCycle);
222  void AdvancePastStalls(SUnit *SU);
223  void EmitNode(SUnit *SU);
224  void ScheduleNodeBottomUp(SUnit*);
225  void CapturePred(SDep *PredEdge);
226  void UnscheduleNodeBottomUp(SUnit*);
227  void RestoreHazardCheckerBottomUp();
228  void BacktrackBottomUp(SUnit*, SUnit*);
229  SUnit *CopyAndMoveSuccessors(SUnit*);
230  void InsertCopiesAndMoveSuccs(SUnit*, unsigned,
231  const TargetRegisterClass*,
232  const TargetRegisterClass*,
234  bool DelayForLiveRegsBottomUp(SUnit*, SmallVectorImpl<unsigned>&);
235 
236  void releaseInterferences(unsigned Reg = 0);
237 
238  SUnit *PickNodeToScheduleBottomUp();
239  void ListScheduleBottomUp();
240 
241  /// CreateNewSUnit - Creates a new SUnit and returns a pointer to it.
242  /// Updates the topological ordering if required.
243  SUnit *CreateNewSUnit(SDNode *N) {
244  unsigned NumSUnits = SUnits.size();
245  SUnit *NewNode = newSUnit(N);
246  // Update the topological ordering.
247  if (NewNode->NodeNum >= NumSUnits)
248  Topo.InitDAGTopologicalSorting();
249  return NewNode;
250  }
251 
252  /// CreateClone - Creates a new SUnit from an existing one.
253  /// Updates the topological ordering if required.
254  SUnit *CreateClone(SUnit *N) {
255  unsigned NumSUnits = SUnits.size();
256  SUnit *NewNode = Clone(N);
257  // Update the topological ordering.
258  if (NewNode->NodeNum >= NumSUnits)
259  Topo.InitDAGTopologicalSorting();
260  return NewNode;
261  }
262 
263  /// forceUnitLatencies - Register-pressure-reducing scheduling doesn't
264  /// need actual latency information but the hybrid scheduler does.
265  bool forceUnitLatencies() const override {
266  return !NeedLatency;
267  }
268 };
269 } // end anonymous namespace
270 
271 /// GetCostForDef - Looks up the register class and cost for a given definition.
272 /// Typically this just means looking up the representative register class,
273 /// but for untyped values (MVT::Untyped) it means inspecting the node's
274 /// opcode to determine what register class is being generated.
275 static void GetCostForDef(const ScheduleDAGSDNodes::RegDefIter &RegDefPos,
276  const TargetLowering *TLI,
277  const TargetInstrInfo *TII,
278  const TargetRegisterInfo *TRI,
279  unsigned &RegClass, unsigned &Cost,
280  const MachineFunction &MF) {
281  MVT VT = RegDefPos.GetValue();
282 
283  // Special handling for untyped values. These values can only come from
284  // the expansion of custom DAG-to-DAG patterns.
285  if (VT == MVT::Untyped) {
286  const SDNode *Node = RegDefPos.GetNode();
287 
288  // Special handling for CopyFromReg of untyped values.
289  if (!Node->isMachineOpcode() && Node->getOpcode() == ISD::CopyFromReg) {
290  unsigned Reg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
291  const TargetRegisterClass *RC = MF.getRegInfo().getRegClass(Reg);
292  RegClass = RC->getID();
293  Cost = 1;
294  return;
295  }
296 
297  unsigned Opcode = Node->getMachineOpcode();
298  if (Opcode == TargetOpcode::REG_SEQUENCE) {
299  unsigned DstRCIdx = cast<ConstantSDNode>(Node->getOperand(0))->getZExtValue();
300  const TargetRegisterClass *RC = TRI->getRegClass(DstRCIdx);
301  RegClass = RC->getID();
302  Cost = 1;
303  return;
304  }
305 
306  unsigned Idx = RegDefPos.GetIdx();
307  const MCInstrDesc Desc = TII->get(Opcode);
308  const TargetRegisterClass *RC = TII->getRegClass(Desc, Idx, TRI, MF);
309  RegClass = RC->getID();
310  // FIXME: Cost arbitrarily set to 1 because there doesn't seem to be a
311  // better way to determine it.
312  Cost = 1;
313  } else {
314  RegClass = TLI->getRepRegClassFor(VT)->getID();
315  Cost = TLI->getRepRegClassCostFor(VT);
316  }
317 }
318 
319 /// Schedule - Schedule the DAG using list scheduling.
320 void ScheduleDAGRRList::Schedule() {
321  DEBUG(dbgs()
322  << "********** List Scheduling BB#" << BB->getNumber()
323  << " '" << BB->getName() << "' **********\n");
324 
325  CurCycle = 0;
326  IssueCount = 0;
327  MinAvailableCycle = DisableSchedCycles ? 0 : UINT_MAX;
328  NumLiveRegs = 0;
329  // Allocate slots for each physical register, plus one for a special register
330  // to track the virtual resource of a calling sequence.
331  LiveRegDefs.resize(TRI->getNumRegs() + 1, nullptr);
332  LiveRegGens.resize(TRI->getNumRegs() + 1, nullptr);
333  CallSeqEndForStart.clear();
334  assert(Interferences.empty() && LRegsMap.empty() && "stale Interferences");
335 
336  // Build the scheduling graph.
337  BuildSchedGraph(nullptr);
338 
339  DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
340  SUnits[su].dumpAll(this));
341  Topo.InitDAGTopologicalSorting();
342 
343  AvailableQueue->initNodes(SUnits);
344 
345  HazardRec->Reset();
346 
347  // Execute the actual scheduling loop.
348  ListScheduleBottomUp();
349 
350  AvailableQueue->releaseState();
351 
352  DEBUG({
353  dbgs() << "*** Final schedule ***\n";
354  dumpSchedule();
355  dbgs() << '\n';
356  });
357 }
358 
359 //===----------------------------------------------------------------------===//
360 // Bottom-Up Scheduling
361 //===----------------------------------------------------------------------===//
362 
363 /// ReleasePred - Decrement the NumSuccsLeft count of a predecessor. Add it to
364 /// the AvailableQueue if the count reaches zero. Also update its cycle bound.
365 void ScheduleDAGRRList::ReleasePred(SUnit *SU, const SDep *PredEdge) {
366  SUnit *PredSU = PredEdge->getSUnit();
367 
368 #ifndef NDEBUG
369  if (PredSU->NumSuccsLeft == 0) {
370  dbgs() << "*** Scheduling failed! ***\n";
371  PredSU->dump(this);
372  dbgs() << " has been released too many times!\n";
373  llvm_unreachable(nullptr);
374  }
375 #endif
376  --PredSU->NumSuccsLeft;
377 
378  if (!forceUnitLatencies()) {
379  // Updating predecessor's height. This is now the cycle when the
380  // predecessor can be scheduled without causing a pipeline stall.
381  PredSU->setHeightToAtLeast(SU->getHeight() + PredEdge->getLatency());
382  }
383 
384  // If all the node's successors are scheduled, this node is ready
385  // to be scheduled. Ignore the special EntrySU node.
386  if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) {
387  PredSU->isAvailable = true;
388 
389  unsigned Height = PredSU->getHeight();
390  if (Height < MinAvailableCycle)
391  MinAvailableCycle = Height;
392 
393  if (isReady(PredSU)) {
394  AvailableQueue->push(PredSU);
395  }
396  // CapturePred and others may have left the node in the pending queue, avoid
397  // adding it twice.
398  else if (!PredSU->isPending) {
399  PredSU->isPending = true;
400  PendingQueue.push_back(PredSU);
401  }
402  }
403 }
404 
405 /// IsChainDependent - Test if Outer is reachable from Inner through
406 /// chain dependencies.
407 static bool IsChainDependent(SDNode *Outer, SDNode *Inner,
408  unsigned NestLevel,
409  const TargetInstrInfo *TII) {
410  SDNode *N = Outer;
411  for (;;) {
412  if (N == Inner)
413  return true;
414  // For a TokenFactor, examine each operand. There may be multiple ways
415  // to get to the CALLSEQ_BEGIN, but we need to find the path with the
416  // most nesting in order to ensure that we find the corresponding match.
417  if (N->getOpcode() == ISD::TokenFactor) {
418  for (const SDValue &Op : N->op_values())
419  if (IsChainDependent(Op.getNode(), Inner, NestLevel, TII))
420  return true;
421  return false;
422  }
423  // Check for a lowered CALLSEQ_BEGIN or CALLSEQ_END.
424  if (N->isMachineOpcode()) {
425  if (N->getMachineOpcode() ==
427  ++NestLevel;
428  } else if (N->getMachineOpcode() ==
430  if (NestLevel == 0)
431  return false;
432  --NestLevel;
433  }
434  }
435  // Otherwise, find the chain and continue climbing.
436  for (const SDValue &Op : N->op_values())
437  if (Op.getValueType() == MVT::Other) {
438  N = Op.getNode();
439  goto found_chain_operand;
440  }
441  return false;
442  found_chain_operand:;
443  if (N->getOpcode() == ISD::EntryToken)
444  return false;
445  }
446 }
447 
448 /// FindCallSeqStart - Starting from the (lowered) CALLSEQ_END node, locate
449 /// the corresponding (lowered) CALLSEQ_BEGIN node.
450 ///
451 /// NestLevel and MaxNested are used in recursion to indcate the current level
452 /// of nesting of CALLSEQ_BEGIN and CALLSEQ_END pairs, as well as the maximum
453 /// level seen so far.
454 ///
455 /// TODO: It would be better to give CALLSEQ_END an explicit operand to point
456 /// to the corresponding CALLSEQ_BEGIN to avoid needing to search for it.
457 static SDNode *
458 FindCallSeqStart(SDNode *N, unsigned &NestLevel, unsigned &MaxNest,
459  const TargetInstrInfo *TII) {
460  for (;;) {
461  // For a TokenFactor, examine each operand. There may be multiple ways
462  // to get to the CALLSEQ_BEGIN, but we need to find the path with the
463  // most nesting in order to ensure that we find the corresponding match.
464  if (N->getOpcode() == ISD::TokenFactor) {
465  SDNode *Best = nullptr;
466  unsigned BestMaxNest = MaxNest;
467  for (const SDValue &Op : N->op_values()) {
468  unsigned MyNestLevel = NestLevel;
469  unsigned MyMaxNest = MaxNest;
470  if (SDNode *New = FindCallSeqStart(Op.getNode(),
471  MyNestLevel, MyMaxNest, TII))
472  if (!Best || (MyMaxNest > BestMaxNest)) {
473  Best = New;
474  BestMaxNest = MyMaxNest;
475  }
476  }
477  assert(Best);
478  MaxNest = BestMaxNest;
479  return Best;
480  }
481  // Check for a lowered CALLSEQ_BEGIN or CALLSEQ_END.
482  if (N->isMachineOpcode()) {
483  if (N->getMachineOpcode() ==
485  ++NestLevel;
486  MaxNest = std::max(MaxNest, NestLevel);
487  } else if (N->getMachineOpcode() ==
489  assert(NestLevel != 0);
490  --NestLevel;
491  if (NestLevel == 0)
492  return N;
493  }
494  }
495  // Otherwise, find the chain and continue climbing.
496  for (const SDValue &Op : N->op_values())
497  if (Op.getValueType() == MVT::Other) {
498  N = Op.getNode();
499  goto found_chain_operand;
500  }
501  return nullptr;
502  found_chain_operand:;
503  if (N->getOpcode() == ISD::EntryToken)
504  return nullptr;
505  }
506 }
507 
508 /// Call ReleasePred for each predecessor, then update register live def/gen.
509 /// Always update LiveRegDefs for a register dependence even if the current SU
510 /// also defines the register. This effectively create one large live range
511 /// across a sequence of two-address node. This is important because the
512 /// entire chain must be scheduled together. Example:
513 ///
514 /// flags = (3) add
515 /// flags = (2) addc flags
516 /// flags = (1) addc flags
517 ///
518 /// results in
519 ///
520 /// LiveRegDefs[flags] = 3
521 /// LiveRegGens[flags] = 1
522 ///
523 /// If (2) addc is unscheduled, then (1) addc must also be unscheduled to avoid
524 /// interference on flags.
525 void ScheduleDAGRRList::ReleasePredecessors(SUnit *SU) {
526  // Bottom up: release predecessors
527  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
528  I != E; ++I) {
529  ReleasePred(SU, &*I);
530  if (I->isAssignedRegDep()) {
531  // This is a physical register dependency and it's impossible or
532  // expensive to copy the register. Make sure nothing that can
533  // clobber the register is scheduled between the predecessor and
534  // this node.
535  SUnit *RegDef = LiveRegDefs[I->getReg()]; (void)RegDef;
536  assert((!RegDef || RegDef == SU || RegDef == I->getSUnit()) &&
537  "interference on register dependence");
538  LiveRegDefs[I->getReg()] = I->getSUnit();
539  if (!LiveRegGens[I->getReg()]) {
540  ++NumLiveRegs;
541  LiveRegGens[I->getReg()] = SU;
542  }
543  }
544  }
545 
546  // If we're scheduling a lowered CALLSEQ_END, find the corresponding
547  // CALLSEQ_BEGIN. Inject an artificial physical register dependence between
548  // these nodes, to prevent other calls from being interscheduled with them.
549  unsigned CallResource = TRI->getNumRegs();
550  if (!LiveRegDefs[CallResource])
551  for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode())
552  if (Node->isMachineOpcode() &&
553  Node->getMachineOpcode() == (unsigned)TII->getCallFrameDestroyOpcode()) {
554  unsigned NestLevel = 0;
555  unsigned MaxNest = 0;
556  SDNode *N = FindCallSeqStart(Node, NestLevel, MaxNest, TII);
557 
558  SUnit *Def = &SUnits[N->getNodeId()];
559  CallSeqEndForStart[Def] = SU;
560 
561  ++NumLiveRegs;
562  LiveRegDefs[CallResource] = Def;
563  LiveRegGens[CallResource] = SU;
564  break;
565  }
566 }
567 
568 /// Check to see if any of the pending instructions are ready to issue. If
569 /// so, add them to the available queue.
570 void ScheduleDAGRRList::ReleasePending() {
571  if (DisableSchedCycles) {
572  assert(PendingQueue.empty() && "pending instrs not allowed in this mode");
573  return;
574  }
575 
576  // If the available queue is empty, it is safe to reset MinAvailableCycle.
577  if (AvailableQueue->empty())
578  MinAvailableCycle = UINT_MAX;
579 
580  // Check to see if any of the pending instructions are ready to issue. If
581  // so, add them to the available queue.
582  for (unsigned i = 0, e = PendingQueue.size(); i != e; ++i) {
583  unsigned ReadyCycle = PendingQueue[i]->getHeight();
584  if (ReadyCycle < MinAvailableCycle)
585  MinAvailableCycle = ReadyCycle;
586 
587  if (PendingQueue[i]->isAvailable) {
588  if (!isReady(PendingQueue[i]))
589  continue;
590  AvailableQueue->push(PendingQueue[i]);
591  }
592  PendingQueue[i]->isPending = false;
593  PendingQueue[i] = PendingQueue.back();
594  PendingQueue.pop_back();
595  --i; --e;
596  }
597 }
598 
599 /// Move the scheduler state forward by the specified number of Cycles.
600 void ScheduleDAGRRList::AdvanceToCycle(unsigned NextCycle) {
601  if (NextCycle <= CurCycle)
602  return;
603 
604  IssueCount = 0;
605  AvailableQueue->setCurCycle(NextCycle);
606  if (!HazardRec->isEnabled()) {
607  // Bypass lots of virtual calls in case of long latency.
608  CurCycle = NextCycle;
609  }
610  else {
611  for (; CurCycle != NextCycle; ++CurCycle) {
612  HazardRec->RecedeCycle();
613  }
614  }
615  // FIXME: Instead of visiting the pending Q each time, set a dirty flag on the
616  // available Q to release pending nodes at least once before popping.
617  ReleasePending();
618 }
619 
620 /// Move the scheduler state forward until the specified node's dependents are
621 /// ready and can be scheduled with no resource conflicts.
622 void ScheduleDAGRRList::AdvancePastStalls(SUnit *SU) {
623  if (DisableSchedCycles)
624  return;
625 
626  // FIXME: Nodes such as CopyFromReg probably should not advance the current
627  // cycle. Otherwise, we can wrongly mask real stalls. If the non-machine node
628  // has predecessors the cycle will be advanced when they are scheduled.
629  // But given the crude nature of modeling latency though such nodes, we
630  // currently need to treat these nodes like real instructions.
631  // if (!SU->getNode() || !SU->getNode()->isMachineOpcode()) return;
632 
633  unsigned ReadyCycle = SU->getHeight();
634 
635  // Bump CurCycle to account for latency. We assume the latency of other
636  // available instructions may be hidden by the stall (not a full pipe stall).
637  // This updates the hazard recognizer's cycle before reserving resources for
638  // this instruction.
639  AdvanceToCycle(ReadyCycle);
640 
641  // Calls are scheduled in their preceding cycle, so don't conflict with
642  // hazards from instructions after the call. EmitNode will reset the
643  // scoreboard state before emitting the call.
644  if (SU->isCall)
645  return;
646 
647  // FIXME: For resource conflicts in very long non-pipelined stages, we
648  // should probably skip ahead here to avoid useless scoreboard checks.
649  int Stalls = 0;
650  while (true) {
652  HazardRec->getHazardType(SU, -Stalls);
653 
655  break;
656 
657  ++Stalls;
658  }
659  AdvanceToCycle(CurCycle + Stalls);
660 }
661 
662 /// Record this SUnit in the HazardRecognizer.
663 /// Does not update CurCycle.
664 void ScheduleDAGRRList::EmitNode(SUnit *SU) {
665  if (!HazardRec->isEnabled())
666  return;
667 
668  // Check for phys reg copy.
669  if (!SU->getNode())
670  return;
671 
672  switch (SU->getNode()->getOpcode()) {
673  default:
674  assert(SU->getNode()->isMachineOpcode() &&
675  "This target-independent node should not be scheduled.");
676  break;
677  case ISD::MERGE_VALUES:
678  case ISD::TokenFactor:
679  case ISD::LIFETIME_START:
680  case ISD::LIFETIME_END:
681  case ISD::CopyToReg:
682  case ISD::CopyFromReg:
683  case ISD::EH_LABEL:
684  // Noops don't affect the scoreboard state. Copies are likely to be
685  // removed.
686  return;
687  case ISD::INLINEASM:
688  // For inline asm, clear the pipeline state.
689  HazardRec->Reset();
690  return;
691  }
692  if (SU->isCall) {
693  // Calls are scheduled with their preceding instructions. For bottom-up
694  // scheduling, clear the pipeline state before emitting.
695  HazardRec->Reset();
696  }
697 
698  HazardRec->EmitInstruction(SU);
699 }
700 
701 static void resetVRegCycle(SUnit *SU);
702 
703 /// ScheduleNodeBottomUp - Add the node to the schedule. Decrement the pending
704 /// count of its predecessors. If a predecessor pending count is zero, add it to
705 /// the Available queue.
706 void ScheduleDAGRRList::ScheduleNodeBottomUp(SUnit *SU) {
707  DEBUG(dbgs() << "\n*** Scheduling [" << CurCycle << "]: ");
708  DEBUG(SU->dump(this));
709 
710 #ifndef NDEBUG
711  if (CurCycle < SU->getHeight())
712  DEBUG(dbgs() << " Height [" << SU->getHeight()
713  << "] pipeline stall!\n");
714 #endif
715 
716  // FIXME: Do not modify node height. It may interfere with
717  // backtracking. Instead add a "ready cycle" to SUnit. Before scheduling the
718  // node its ready cycle can aid heuristics, and after scheduling it can
719  // indicate the scheduled cycle.
720  SU->setHeightToAtLeast(CurCycle);
721 
722  // Reserve resources for the scheduled instruction.
723  EmitNode(SU);
724 
725  Sequence.push_back(SU);
726 
727  AvailableQueue->scheduledNode(SU);
728 
729  // If HazardRec is disabled, and each inst counts as one cycle, then
730  // advance CurCycle before ReleasePredecessors to avoid useless pushes to
731  // PendingQueue for schedulers that implement HasReadyFilter.
732  if (!HazardRec->isEnabled() && AvgIPC < 2)
733  AdvanceToCycle(CurCycle + 1);
734 
735  // Update liveness of predecessors before successors to avoid treating a
736  // two-address node as a live range def.
737  ReleasePredecessors(SU);
738 
739  // Release all the implicit physical register defs that are live.
740  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
741  I != E; ++I) {
742  // LiveRegDegs[I->getReg()] != SU when SU is a two-address node.
743  if (I->isAssignedRegDep() && LiveRegDefs[I->getReg()] == SU) {
744  assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!");
745  --NumLiveRegs;
746  LiveRegDefs[I->getReg()] = nullptr;
747  LiveRegGens[I->getReg()] = nullptr;
748  releaseInterferences(I->getReg());
749  }
750  }
751  // Release the special call resource dependence, if this is the beginning
752  // of a call.
753  unsigned CallResource = TRI->getNumRegs();
754  if (LiveRegDefs[CallResource] == SU)
755  for (const SDNode *SUNode = SU->getNode(); SUNode;
756  SUNode = SUNode->getGluedNode()) {
757  if (SUNode->isMachineOpcode() &&
758  SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameSetupOpcode()) {
759  assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!");
760  --NumLiveRegs;
761  LiveRegDefs[CallResource] = nullptr;
762  LiveRegGens[CallResource] = nullptr;
763  releaseInterferences(CallResource);
764  }
765  }
766 
767  resetVRegCycle(SU);
768 
769  SU->isScheduled = true;
770 
771  // Conditions under which the scheduler should eagerly advance the cycle:
772  // (1) No available instructions
773  // (2) All pipelines full, so available instructions must have hazards.
774  //
775  // If HazardRec is disabled, the cycle was pre-advanced before calling
776  // ReleasePredecessors. In that case, IssueCount should remain 0.
777  //
778  // Check AvailableQueue after ReleasePredecessors in case of zero latency.
779  if (HazardRec->isEnabled() || AvgIPC > 1) {
780  if (SU->getNode() && SU->getNode()->isMachineOpcode())
781  ++IssueCount;
782  if ((HazardRec->isEnabled() && HazardRec->atIssueLimit())
783  || (!HazardRec->isEnabled() && IssueCount == AvgIPC))
784  AdvanceToCycle(CurCycle + 1);
785  }
786 }
787 
788 /// CapturePred - This does the opposite of ReleasePred. Since SU is being
789 /// unscheduled, incrcease the succ left count of its predecessors. Remove
790 /// them from AvailableQueue if necessary.
791 void ScheduleDAGRRList::CapturePred(SDep *PredEdge) {
792  SUnit *PredSU = PredEdge->getSUnit();
793  if (PredSU->isAvailable) {
794  PredSU->isAvailable = false;
795  if (!PredSU->isPending)
796  AvailableQueue->remove(PredSU);
797  }
798 
799  assert(PredSU->NumSuccsLeft < UINT_MAX && "NumSuccsLeft will overflow!");
800  ++PredSU->NumSuccsLeft;
801 }
802 
803 /// UnscheduleNodeBottomUp - Remove the node from the schedule, update its and
804 /// its predecessor states to reflect the change.
805 void ScheduleDAGRRList::UnscheduleNodeBottomUp(SUnit *SU) {
806  DEBUG(dbgs() << "*** Unscheduling [" << SU->getHeight() << "]: ");
807  DEBUG(SU->dump(this));
808 
809  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
810  I != E; ++I) {
811  CapturePred(&*I);
812  if (I->isAssignedRegDep() && SU == LiveRegGens[I->getReg()]){
813  assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!");
814  assert(LiveRegDefs[I->getReg()] == I->getSUnit() &&
815  "Physical register dependency violated?");
816  --NumLiveRegs;
817  LiveRegDefs[I->getReg()] = nullptr;
818  LiveRegGens[I->getReg()] = nullptr;
819  releaseInterferences(I->getReg());
820  }
821  }
822 
823  // Reclaim the special call resource dependence, if this is the beginning
824  // of a call.
825  unsigned CallResource = TRI->getNumRegs();
826  for (const SDNode *SUNode = SU->getNode(); SUNode;
827  SUNode = SUNode->getGluedNode()) {
828  if (SUNode->isMachineOpcode() &&
829  SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameSetupOpcode()) {
830  ++NumLiveRegs;
831  LiveRegDefs[CallResource] = SU;
832  LiveRegGens[CallResource] = CallSeqEndForStart[SU];
833  }
834  }
835 
836  // Release the special call resource dependence, if this is the end
837  // of a call.
838  if (LiveRegGens[CallResource] == SU)
839  for (const SDNode *SUNode = SU->getNode(); SUNode;
840  SUNode = SUNode->getGluedNode()) {
841  if (SUNode->isMachineOpcode() &&
842  SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameDestroyOpcode()) {
843  assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!");
844  --NumLiveRegs;
845  LiveRegDefs[CallResource] = nullptr;
846  LiveRegGens[CallResource] = nullptr;
847  releaseInterferences(CallResource);
848  }
849  }
850 
851  for (auto &Succ : SU->Succs) {
852  if (Succ.isAssignedRegDep()) {
853  auto Reg = Succ.getReg();
854  if (!LiveRegDefs[Reg])
855  ++NumLiveRegs;
856  // This becomes the nearest def. Note that an earlier def may still be
857  // pending if this is a two-address node.
858  LiveRegDefs[Reg] = SU;
859 
860  // Update LiveRegGen only if was empty before this unscheduling.
861  // This is to avoid incorrect updating LiveRegGen set in previous run.
862  if (!LiveRegGens[Reg]) {
863  // Find the successor with the lowest height.
864  LiveRegGens[Reg] = Succ.getSUnit();
865  for (auto &Succ2 : SU->Succs) {
866  if (Succ2.isAssignedRegDep() && Succ2.getReg() == Reg &&
867  Succ2.getSUnit()->getHeight() < LiveRegGens[Reg]->getHeight())
868  LiveRegGens[Reg] = Succ2.getSUnit();
869  }
870  }
871  }
872  }
873  if (SU->getHeight() < MinAvailableCycle)
874  MinAvailableCycle = SU->getHeight();
875 
876  SU->setHeightDirty();
877  SU->isScheduled = false;
878  SU->isAvailable = true;
879  if (!DisableSchedCycles && AvailableQueue->hasReadyFilter()) {
880  // Don't make available until backtracking is complete.
881  SU->isPending = true;
882  PendingQueue.push_back(SU);
883  }
884  else {
885  AvailableQueue->push(SU);
886  }
887  AvailableQueue->unscheduledNode(SU);
888 }
889 
890 /// After backtracking, the hazard checker needs to be restored to a state
891 /// corresponding the current cycle.
892 void ScheduleDAGRRList::RestoreHazardCheckerBottomUp() {
893  HazardRec->Reset();
894 
895  unsigned LookAhead = std::min((unsigned)Sequence.size(),
896  HazardRec->getMaxLookAhead());
897  if (LookAhead == 0)
898  return;
899 
900  std::vector<SUnit*>::const_iterator I = (Sequence.end() - LookAhead);
901  unsigned HazardCycle = (*I)->getHeight();
902  for (std::vector<SUnit*>::const_iterator E = Sequence.end(); I != E; ++I) {
903  SUnit *SU = *I;
904  for (; SU->getHeight() > HazardCycle; ++HazardCycle) {
905  HazardRec->RecedeCycle();
906  }
907  EmitNode(SU);
908  }
909 }
910 
911 /// BacktrackBottomUp - Backtrack scheduling to a previous cycle specified in
912 /// BTCycle in order to schedule a specific node.
913 void ScheduleDAGRRList::BacktrackBottomUp(SUnit *SU, SUnit *BtSU) {
914  SUnit *OldSU = Sequence.back();
915  while (true) {
916  Sequence.pop_back();
917  // FIXME: use ready cycle instead of height
918  CurCycle = OldSU->getHeight();
919  UnscheduleNodeBottomUp(OldSU);
920  AvailableQueue->setCurCycle(CurCycle);
921  if (OldSU == BtSU)
922  break;
923  OldSU = Sequence.back();
924  }
925 
926  assert(!SU->isSucc(OldSU) && "Something is wrong!");
927 
928  RestoreHazardCheckerBottomUp();
929 
930  ReleasePending();
931 
932  ++NumBacktracks;
933 }
934 
935 static bool isOperandOf(const SUnit *SU, SDNode *N) {
936  for (const SDNode *SUNode = SU->getNode(); SUNode;
937  SUNode = SUNode->getGluedNode()) {
938  if (SUNode->isOperandOf(N))
939  return true;
940  }
941  return false;
942 }
943 
944 /// CopyAndMoveSuccessors - Clone the specified node and move its scheduled
945 /// successors to the newly created node.
946 SUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) {
947  SDNode *N = SU->getNode();
948  if (!N)
949  return nullptr;
950 
951  if (SU->getNode()->getGluedNode())
952  return nullptr;
953 
954  SUnit *NewSU;
955  bool TryUnfold = false;
956  for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
957  MVT VT = N->getSimpleValueType(i);
958  if (VT == MVT::Glue)
959  return nullptr;
960  else if (VT == MVT::Other)
961  TryUnfold = true;
962  }
963  for (const SDValue &Op : N->op_values()) {
964  MVT VT = Op.getNode()->getSimpleValueType(Op.getResNo());
965  if (VT == MVT::Glue)
966  return nullptr;
967  }
968 
969  if (TryUnfold) {
970  SmallVector<SDNode*, 2> NewNodes;
971  if (!TII->unfoldMemoryOperand(*DAG, N, NewNodes))
972  return nullptr;
973 
974  // unfolding an x86 DEC64m operation results in store, dec, load which
975  // can't be handled here so quit
976  if (NewNodes.size() == 3)
977  return nullptr;
978 
979  DEBUG(dbgs() << "Unfolding SU #" << SU->NodeNum << "\n");
980  assert(NewNodes.size() == 2 && "Expected a load folding node!");
981 
982  N = NewNodes[1];
983  SDNode *LoadNode = NewNodes[0];
984  unsigned NumVals = N->getNumValues();
985  unsigned OldNumVals = SU->getNode()->getNumValues();
986  for (unsigned i = 0; i != NumVals; ++i)
987  DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i));
988  DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals-1),
989  SDValue(LoadNode, 1));
990 
991  // LoadNode may already exist. This can happen when there is another
992  // load from the same location and producing the same type of value
993  // but it has different alignment or volatileness.
994  bool isNewLoad = true;
995  SUnit *LoadSU;
996  if (LoadNode->getNodeId() != -1) {
997  LoadSU = &SUnits[LoadNode->getNodeId()];
998  isNewLoad = false;
999  } else {
1000  LoadSU = CreateNewSUnit(LoadNode);
1001  LoadNode->setNodeId(LoadSU->NodeNum);
1002 
1003  InitNumRegDefsLeft(LoadSU);
1004  computeLatency(LoadSU);
1005  }
1006 
1007  SUnit *NewSU = CreateNewSUnit(N);
1008  assert(N->getNodeId() == -1 && "Node already inserted!");
1009  N->setNodeId(NewSU->NodeNum);
1010 
1011  const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
1012  for (unsigned i = 0; i != MCID.getNumOperands(); ++i) {
1013  if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) {
1014  NewSU->isTwoAddress = true;
1015  break;
1016  }
1017  }
1018  if (MCID.isCommutable())
1019  NewSU->isCommutable = true;
1020 
1021  InitNumRegDefsLeft(NewSU);
1022  computeLatency(NewSU);
1023 
1024  // Record all the edges to and from the old SU, by category.
1025  SmallVector<SDep, 4> ChainPreds;
1026  SmallVector<SDep, 4> ChainSuccs;
1027  SmallVector<SDep, 4> LoadPreds;
1028  SmallVector<SDep, 4> NodePreds;
1029  SmallVector<SDep, 4> NodeSuccs;
1030  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
1031  I != E; ++I) {
1032  if (I->isCtrl())
1033  ChainPreds.push_back(*I);
1034  else if (isOperandOf(I->getSUnit(), LoadNode))
1035  LoadPreds.push_back(*I);
1036  else
1037  NodePreds.push_back(*I);
1038  }
1039  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
1040  I != E; ++I) {
1041  if (I->isCtrl())
1042  ChainSuccs.push_back(*I);
1043  else
1044  NodeSuccs.push_back(*I);
1045  }
1046 
1047  // Now assign edges to the newly-created nodes.
1048  for (unsigned i = 0, e = ChainPreds.size(); i != e; ++i) {
1049  const SDep &Pred = ChainPreds[i];
1050  RemovePred(SU, Pred);
1051  if (isNewLoad)
1052  AddPred(LoadSU, Pred);
1053  }
1054  for (unsigned i = 0, e = LoadPreds.size(); i != e; ++i) {
1055  const SDep &Pred = LoadPreds[i];
1056  RemovePred(SU, Pred);
1057  if (isNewLoad)
1058  AddPred(LoadSU, Pred);
1059  }
1060  for (unsigned i = 0, e = NodePreds.size(); i != e; ++i) {
1061  const SDep &Pred = NodePreds[i];
1062  RemovePred(SU, Pred);
1063  AddPred(NewSU, Pred);
1064  }
1065  for (unsigned i = 0, e = NodeSuccs.size(); i != e; ++i) {
1066  SDep D = NodeSuccs[i];
1067  SUnit *SuccDep = D.getSUnit();
1068  D.setSUnit(SU);
1069  RemovePred(SuccDep, D);
1070  D.setSUnit(NewSU);
1071  AddPred(SuccDep, D);
1072  // Balance register pressure.
1073  if (AvailableQueue->tracksRegPressure() && SuccDep->isScheduled
1074  && !D.isCtrl() && NewSU->NumRegDefsLeft > 0)
1075  --NewSU->NumRegDefsLeft;
1076  }
1077  for (unsigned i = 0, e = ChainSuccs.size(); i != e; ++i) {
1078  SDep D = ChainSuccs[i];
1079  SUnit *SuccDep = D.getSUnit();
1080  D.setSUnit(SU);
1081  RemovePred(SuccDep, D);
1082  if (isNewLoad) {
1083  D.setSUnit(LoadSU);
1084  AddPred(SuccDep, D);
1085  }
1086  }
1087 
1088  // Add a data dependency to reflect that NewSU reads the value defined
1089  // by LoadSU.
1090  SDep D(LoadSU, SDep::Data, 0);
1091  D.setLatency(LoadSU->Latency);
1092  AddPred(NewSU, D);
1093 
1094  if (isNewLoad)
1095  AvailableQueue->addNode(LoadSU);
1096  AvailableQueue->addNode(NewSU);
1097 
1098  ++NumUnfolds;
1099 
1100  if (NewSU->NumSuccsLeft == 0) {
1101  NewSU->isAvailable = true;
1102  return NewSU;
1103  }
1104  SU = NewSU;
1105  }
1106 
1107  DEBUG(dbgs() << " Duplicating SU #" << SU->NodeNum << "\n");
1108  NewSU = CreateClone(SU);
1109 
1110  // New SUnit has the exact same predecessors.
1111  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
1112  I != E; ++I)
1113  if (!I->isArtificial())
1114  AddPred(NewSU, *I);
1115 
1116  // Only copy scheduled successors. Cut them from old node's successor
1117  // list and move them over.
1119  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
1120  I != E; ++I) {
1121  if (I->isArtificial())
1122  continue;
1123  SUnit *SuccSU = I->getSUnit();
1124  if (SuccSU->isScheduled) {
1125  SDep D = *I;
1126  D.setSUnit(NewSU);
1127  AddPred(SuccSU, D);
1128  D.setSUnit(SU);
1129  DelDeps.push_back(std::make_pair(SuccSU, D));
1130  }
1131  }
1132  for (unsigned i = 0, e = DelDeps.size(); i != e; ++i)
1133  RemovePred(DelDeps[i].first, DelDeps[i].second);
1134 
1135  AvailableQueue->updateNode(SU);
1136  AvailableQueue->addNode(NewSU);
1137 
1138  ++NumDups;
1139  return NewSU;
1140 }
1141 
1142 /// InsertCopiesAndMoveSuccs - Insert register copies and move all
1143 /// scheduled successors of the given SUnit to the last copy.
1144 void ScheduleDAGRRList::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg,
1145  const TargetRegisterClass *DestRC,
1146  const TargetRegisterClass *SrcRC,
1148  SUnit *CopyFromSU = CreateNewSUnit(nullptr);
1149  CopyFromSU->CopySrcRC = SrcRC;
1150  CopyFromSU->CopyDstRC = DestRC;
1151 
1152  SUnit *CopyToSU = CreateNewSUnit(nullptr);
1153  CopyToSU->CopySrcRC = DestRC;
1154  CopyToSU->CopyDstRC = SrcRC;
1155 
1156  // Only copy scheduled successors. Cut them from old node's successor
1157  // list and move them over.
1159  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
1160  I != E; ++I) {
1161  if (I->isArtificial())
1162  continue;
1163  SUnit *SuccSU = I->getSUnit();
1164  if (SuccSU->isScheduled) {
1165  SDep D = *I;
1166  D.setSUnit(CopyToSU);
1167  AddPred(SuccSU, D);
1168  DelDeps.push_back(std::make_pair(SuccSU, *I));
1169  }
1170  else {
1171  // Avoid scheduling the def-side copy before other successors. Otherwise
1172  // we could introduce another physreg interference on the copy and
1173  // continue inserting copies indefinitely.
1174  AddPred(SuccSU, SDep(CopyFromSU, SDep::Artificial));
1175  }
1176  }
1177  for (unsigned i = 0, e = DelDeps.size(); i != e; ++i)
1178  RemovePred(DelDeps[i].first, DelDeps[i].second);
1179 
1180  SDep FromDep(SU, SDep::Data, Reg);
1181  FromDep.setLatency(SU->Latency);
1182  AddPred(CopyFromSU, FromDep);
1183  SDep ToDep(CopyFromSU, SDep::Data, 0);
1184  ToDep.setLatency(CopyFromSU->Latency);
1185  AddPred(CopyToSU, ToDep);
1186 
1187  AvailableQueue->updateNode(SU);
1188  AvailableQueue->addNode(CopyFromSU);
1189  AvailableQueue->addNode(CopyToSU);
1190  Copies.push_back(CopyFromSU);
1191  Copies.push_back(CopyToSU);
1192 
1193  ++NumPRCopies;
1194 }
1195 
1196 /// getPhysicalRegisterVT - Returns the ValueType of the physical register
1197 /// definition of the specified node.
1198 /// FIXME: Move to SelectionDAG?
1199 static MVT getPhysicalRegisterVT(SDNode *N, unsigned Reg,
1200  const TargetInstrInfo *TII) {
1201  unsigned NumRes;
1202  if (N->getOpcode() == ISD::CopyFromReg) {
1203  // CopyFromReg has: "chain, Val, glue" so operand 1 gives the type.
1204  NumRes = 1;
1205  } else {
1206  const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
1207  assert(MCID.ImplicitDefs && "Physical reg def must be in implicit def list!");
1208  NumRes = MCID.getNumDefs();
1209  for (const uint16_t *ImpDef = MCID.getImplicitDefs(); *ImpDef; ++ImpDef) {
1210  if (Reg == *ImpDef)
1211  break;
1212  ++NumRes;
1213  }
1214  }
1215  return N->getSimpleValueType(NumRes);
1216 }
1217 
1218 /// CheckForLiveRegDef - Return true and update live register vector if the
1219 /// specified register def of the specified SUnit clobbers any "live" registers.
1220 static void CheckForLiveRegDef(SUnit *SU, unsigned Reg,
1221  std::vector<SUnit*> &LiveRegDefs,
1222  SmallSet<unsigned, 4> &RegAdded,
1224  const TargetRegisterInfo *TRI) {
1225  for (MCRegAliasIterator AliasI(Reg, TRI, true); AliasI.isValid(); ++AliasI) {
1226 
1227  // Check if Ref is live.
1228  if (!LiveRegDefs[*AliasI]) continue;
1229 
1230  // Allow multiple uses of the same def.
1231  if (LiveRegDefs[*AliasI] == SU) continue;
1232 
1233  // Add Reg to the set of interfering live regs.
1234  if (RegAdded.insert(*AliasI).second) {
1235  LRegs.push_back(*AliasI);
1236  }
1237  }
1238 }
1239 
1240 /// CheckForLiveRegDefMasked - Check for any live physregs that are clobbered
1241 /// by RegMask, and add them to LRegs.
1242 static void CheckForLiveRegDefMasked(SUnit *SU, const uint32_t *RegMask,
1243  std::vector<SUnit*> &LiveRegDefs,
1244  SmallSet<unsigned, 4> &RegAdded,
1245  SmallVectorImpl<unsigned> &LRegs) {
1246  // Look at all live registers. Skip Reg0 and the special CallResource.
1247  for (unsigned i = 1, e = LiveRegDefs.size()-1; i != e; ++i) {
1248  if (!LiveRegDefs[i]) continue;
1249  if (LiveRegDefs[i] == SU) continue;
1250  if (!MachineOperand::clobbersPhysReg(RegMask, i)) continue;
1251  if (RegAdded.insert(i).second)
1252  LRegs.push_back(i);
1253  }
1254 }
1255 
1256 /// getNodeRegMask - Returns the register mask attached to an SDNode, if any.
1257 static const uint32_t *getNodeRegMask(const SDNode *N) {
1258  for (const SDValue &Op : N->op_values())
1259  if (const auto *RegOp = dyn_cast<RegisterMaskSDNode>(Op.getNode()))
1260  return RegOp->getRegMask();
1261  return nullptr;
1262 }
1263 
1264 /// DelayForLiveRegsBottomUp - Returns true if it is necessary to delay
1265 /// scheduling of the given node to satisfy live physical register dependencies.
1266 /// If the specific node is the last one that's available to schedule, do
1267 /// whatever is necessary (i.e. backtracking or cloning) to make it possible.
1268 bool ScheduleDAGRRList::
1269 DelayForLiveRegsBottomUp(SUnit *SU, SmallVectorImpl<unsigned> &LRegs) {
1270  if (NumLiveRegs == 0)
1271  return false;
1272 
1273  SmallSet<unsigned, 4> RegAdded;
1274  // If this node would clobber any "live" register, then it's not ready.
1275  //
1276  // If SU is the currently live definition of the same register that it uses,
1277  // then we are free to schedule it.
1278  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
1279  I != E; ++I) {
1280  if (I->isAssignedRegDep() && LiveRegDefs[I->getReg()] != SU)
1281  CheckForLiveRegDef(I->getSUnit(), I->getReg(), LiveRegDefs,
1282  RegAdded, LRegs, TRI);
1283  }
1284 
1285  for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) {
1286  if (Node->getOpcode() == ISD::INLINEASM) {
1287  // Inline asm can clobber physical defs.
1288  unsigned NumOps = Node->getNumOperands();
1289  if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
1290  --NumOps; // Ignore the glue operand.
1291 
1292  for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
1293  unsigned Flags =
1294  cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
1295  unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
1296 
1297  ++i; // Skip the ID value.
1298  if (InlineAsm::isRegDefKind(Flags) ||
1300  InlineAsm::isClobberKind(Flags)) {
1301  // Check for def of register or earlyclobber register.
1302  for (; NumVals; --NumVals, ++i) {
1303  unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
1305  CheckForLiveRegDef(SU, Reg, LiveRegDefs, RegAdded, LRegs, TRI);
1306  }
1307  } else
1308  i += NumVals;
1309  }
1310  continue;
1311  }
1312 
1313  if (!Node->isMachineOpcode())
1314  continue;
1315  // If we're in the middle of scheduling a call, don't begin scheduling
1316  // another call. Also, don't allow any physical registers to be live across
1317  // the call.
1318  if (Node->getMachineOpcode() == (unsigned)TII->getCallFrameDestroyOpcode()) {
1319  // Check the special calling-sequence resource.
1320  unsigned CallResource = TRI->getNumRegs();
1321  if (LiveRegDefs[CallResource]) {
1322  SDNode *Gen = LiveRegGens[CallResource]->getNode();
1323  while (SDNode *Glued = Gen->getGluedNode())
1324  Gen = Glued;
1325  if (!IsChainDependent(Gen, Node, 0, TII) &&
1326  RegAdded.insert(CallResource).second)
1327  LRegs.push_back(CallResource);
1328  }
1329  }
1330  if (const uint32_t *RegMask = getNodeRegMask(Node))
1331  CheckForLiveRegDefMasked(SU, RegMask, LiveRegDefs, RegAdded, LRegs);
1332 
1333  const MCInstrDesc &MCID = TII->get(Node->getMachineOpcode());
1334  if (!MCID.ImplicitDefs)
1335  continue;
1336  for (const uint16_t *Reg = MCID.getImplicitDefs(); *Reg; ++Reg)
1337  CheckForLiveRegDef(SU, *Reg, LiveRegDefs, RegAdded, LRegs, TRI);
1338  }
1339 
1340  return !LRegs.empty();
1341 }
1342 
1343 void ScheduleDAGRRList::releaseInterferences(unsigned Reg) {
1344  // Add the nodes that aren't ready back onto the available list.
1345  for (unsigned i = Interferences.size(); i > 0; --i) {
1346  SUnit *SU = Interferences[i-1];
1347  LRegsMapT::iterator LRegsPos = LRegsMap.find(SU);
1348  if (Reg) {
1349  SmallVectorImpl<unsigned> &LRegs = LRegsPos->second;
1350  if (std::find(LRegs.begin(), LRegs.end(), Reg) == LRegs.end())
1351  continue;
1352  }
1353  SU->isPending = false;
1354  // The interfering node may no longer be available due to backtracking.
1355  // Furthermore, it may have been made available again, in which case it is
1356  // now already in the AvailableQueue.
1357  if (SU->isAvailable && !SU->NodeQueueId) {
1358  DEBUG(dbgs() << " Repushing SU #" << SU->NodeNum << '\n');
1359  AvailableQueue->push(SU);
1360  }
1361  if (i < Interferences.size())
1362  Interferences[i-1] = Interferences.back();
1363  Interferences.pop_back();
1364  LRegsMap.erase(LRegsPos);
1365  }
1366 }
1367 
1368 /// Return a node that can be scheduled in this cycle. Requirements:
1369 /// (1) Ready: latency has been satisfied
1370 /// (2) No Hazards: resources are available
1371 /// (3) No Interferences: may unschedule to break register interferences.
1372 SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() {
1373  SUnit *CurSU = AvailableQueue->empty() ? nullptr : AvailableQueue->pop();
1374  while (CurSU) {
1376  if (!DelayForLiveRegsBottomUp(CurSU, LRegs))
1377  break;
1378  DEBUG(dbgs() << " Interfering reg " <<
1379  (LRegs[0] == TRI->getNumRegs() ? "CallResource"
1380  : TRI->getName(LRegs[0]))
1381  << " SU #" << CurSU->NodeNum << '\n');
1382  std::pair<LRegsMapT::iterator, bool> LRegsPair =
1383  LRegsMap.insert(std::make_pair(CurSU, LRegs));
1384  if (LRegsPair.second) {
1385  CurSU->isPending = true; // This SU is not in AvailableQueue right now.
1386  Interferences.push_back(CurSU);
1387  }
1388  else {
1389  assert(CurSU->isPending && "Interferences are pending");
1390  // Update the interference with current live regs.
1391  LRegsPair.first->second = LRegs;
1392  }
1393  CurSU = AvailableQueue->pop();
1394  }
1395  if (CurSU)
1396  return CurSU;
1397 
1398  // All candidates are delayed due to live physical reg dependencies.
1399  // Try backtracking, code duplication, or inserting cross class copies
1400  // to resolve it.
1401  for (unsigned i = 0, e = Interferences.size(); i != e; ++i) {
1402  SUnit *TrySU = Interferences[i];
1403  SmallVectorImpl<unsigned> &LRegs = LRegsMap[TrySU];
1404 
1405  // Try unscheduling up to the point where it's safe to schedule
1406  // this node.
1407  SUnit *BtSU = nullptr;
1408  unsigned LiveCycle = UINT_MAX;
1409  for (unsigned j = 0, ee = LRegs.size(); j != ee; ++j) {
1410  unsigned Reg = LRegs[j];
1411  if (LiveRegGens[Reg]->getHeight() < LiveCycle) {
1412  BtSU = LiveRegGens[Reg];
1413  LiveCycle = BtSU->getHeight();
1414  }
1415  }
1416  if (!WillCreateCycle(TrySU, BtSU)) {
1417  // BacktrackBottomUp mutates Interferences!
1418  BacktrackBottomUp(TrySU, BtSU);
1419 
1420  // Force the current node to be scheduled before the node that
1421  // requires the physical reg dep.
1422  if (BtSU->isAvailable) {
1423  BtSU->isAvailable = false;
1424  if (!BtSU->isPending)
1425  AvailableQueue->remove(BtSU);
1426  }
1427  DEBUG(dbgs() << "ARTIFICIAL edge from SU(" << BtSU->NodeNum << ") to SU("
1428  << TrySU->NodeNum << ")\n");
1429  AddPred(TrySU, SDep(BtSU, SDep::Artificial));
1430 
1431  // If one or more successors has been unscheduled, then the current
1432  // node is no longer available.
1433  if (!TrySU->isAvailable || !TrySU->NodeQueueId)
1434  CurSU = AvailableQueue->pop();
1435  else {
1436  // Available and in AvailableQueue
1437  AvailableQueue->remove(TrySU);
1438  CurSU = TrySU;
1439  }
1440  // Interferences has been mutated. We must break.
1441  break;
1442  }
1443  }
1444 
1445  if (!CurSU) {
1446  // Can't backtrack. If it's too expensive to copy the value, then try
1447  // duplicate the nodes that produces these "too expensive to copy"
1448  // values to break the dependency. In case even that doesn't work,
1449  // insert cross class copies.
1450  // If it's not too expensive, i.e. cost != -1, issue copies.
1451  SUnit *TrySU = Interferences[0];
1452  SmallVectorImpl<unsigned> &LRegs = LRegsMap[TrySU];
1453  assert(LRegs.size() == 1 && "Can't handle this yet!");
1454  unsigned Reg = LRegs[0];
1455  SUnit *LRDef = LiveRegDefs[Reg];
1456  MVT VT = getPhysicalRegisterVT(LRDef->getNode(), Reg, TII);
1457  const TargetRegisterClass *RC =
1458  TRI->getMinimalPhysRegClass(Reg, VT);
1459  const TargetRegisterClass *DestRC = TRI->getCrossCopyRegClass(RC);
1460 
1461  // If cross copy register class is the same as RC, then it must be possible
1462  // copy the value directly. Do not try duplicate the def.
1463  // If cross copy register class is not the same as RC, then it's possible to
1464  // copy the value but it require cross register class copies and it is
1465  // expensive.
1466  // If cross copy register class is null, then it's not possible to copy
1467  // the value at all.
1468  SUnit *NewDef = nullptr;
1469  if (DestRC != RC) {
1470  NewDef = CopyAndMoveSuccessors(LRDef);
1471  if (!DestRC && !NewDef)
1472  report_fatal_error("Can't handle live physical register dependency!");
1473  }
1474  if (!NewDef) {
1475  // Issue copies, these can be expensive cross register class copies.
1477  InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
1478  DEBUG(dbgs() << " Adding an edge from SU #" << TrySU->NodeNum
1479  << " to SU #" << Copies.front()->NodeNum << "\n");
1480  AddPred(TrySU, SDep(Copies.front(), SDep::Artificial));
1481  NewDef = Copies.back();
1482  }
1483 
1484  DEBUG(dbgs() << " Adding an edge from SU #" << NewDef->NodeNum
1485  << " to SU #" << TrySU->NodeNum << "\n");
1486  LiveRegDefs[Reg] = NewDef;
1487  AddPred(NewDef, SDep(TrySU, SDep::Artificial));
1488  TrySU->isAvailable = false;
1489  CurSU = NewDef;
1490  }
1491  assert(CurSU && "Unable to resolve live physical register dependencies!");
1492  return CurSU;
1493 }
1494 
1495 /// ListScheduleBottomUp - The main loop of list scheduling for bottom-up
1496 /// schedulers.
1497 void ScheduleDAGRRList::ListScheduleBottomUp() {
1498  // Release any predecessors of the special Exit node.
1499  ReleasePredecessors(&ExitSU);
1500 
1501  // Add root to Available queue.
1502  if (!SUnits.empty()) {
1503  SUnit *RootSU = &SUnits[DAG->getRoot().getNode()->getNodeId()];
1504  assert(RootSU->Succs.empty() && "Graph root shouldn't have successors!");
1505  RootSU->isAvailable = true;
1506  AvailableQueue->push(RootSU);
1507  }
1508 
1509  // While Available queue is not empty, grab the node with the highest
1510  // priority. If it is not ready put it back. Schedule the node.
1511  Sequence.reserve(SUnits.size());
1512  while (!AvailableQueue->empty() || !Interferences.empty()) {
1513  DEBUG(dbgs() << "\nExamining Available:\n";
1514  AvailableQueue->dump(this));
1515 
1516  // Pick the best node to schedule taking all constraints into
1517  // consideration.
1518  SUnit *SU = PickNodeToScheduleBottomUp();
1519 
1520  AdvancePastStalls(SU);
1521 
1522  ScheduleNodeBottomUp(SU);
1523 
1524  while (AvailableQueue->empty() && !PendingQueue.empty()) {
1525  // Advance the cycle to free resources. Skip ahead to the next ready SU.
1526  assert(MinAvailableCycle < UINT_MAX && "MinAvailableCycle uninitialized");
1527  AdvanceToCycle(std::max(CurCycle + 1, MinAvailableCycle));
1528  }
1529  }
1530 
1531  // Reverse the order if it is bottom up.
1532  std::reverse(Sequence.begin(), Sequence.end());
1533 
1534 #ifndef NDEBUG
1535  VerifyScheduledSequence(/*isBottomUp=*/true);
1536 #endif
1537 }
1538 
1539 //===----------------------------------------------------------------------===//
1540 // RegReductionPriorityQueue Definition
1541 //===----------------------------------------------------------------------===//
1542 //
1543 // This is a SchedulingPriorityQueue that schedules using Sethi Ullman numbers
1544 // to reduce register pressure.
1545 //
1546 namespace {
1547 class RegReductionPQBase;
1548 
1549 struct queue_sort : public std::binary_function<SUnit*, SUnit*, bool> {
1550  bool isReady(SUnit* SU, unsigned CurCycle) const { return true; }
1551 };
1552 
1553 #ifndef NDEBUG
1554 template<class SF>
1555 struct reverse_sort : public queue_sort {
1556  SF &SortFunc;
1557  reverse_sort(SF &sf) : SortFunc(sf) {}
1558 
1559  bool operator()(SUnit* left, SUnit* right) const {
1560  // reverse left/right rather than simply !SortFunc(left, right)
1561  // to expose different paths in the comparison logic.
1562  return SortFunc(right, left);
1563  }
1564 };
1565 #endif // NDEBUG
1566 
1567 /// bu_ls_rr_sort - Priority function for bottom up register pressure
1568 // reduction scheduler.
1569 struct bu_ls_rr_sort : public queue_sort {
1570  enum {
1571  IsBottomUp = true,
1572  HasReadyFilter = false
1573  };
1574 
1575  RegReductionPQBase *SPQ;
1576  bu_ls_rr_sort(RegReductionPQBase *spq) : SPQ(spq) {}
1577 
1578  bool operator()(SUnit* left, SUnit* right) const;
1579 };
1580 
1581 // src_ls_rr_sort - Priority function for source order scheduler.
1582 struct src_ls_rr_sort : public queue_sort {
1583  enum {
1584  IsBottomUp = true,
1585  HasReadyFilter = false
1586  };
1587 
1588  RegReductionPQBase *SPQ;
1589  src_ls_rr_sort(RegReductionPQBase *spq)
1590  : SPQ(spq) {}
1591 
1592  bool operator()(SUnit* left, SUnit* right) const;
1593 };
1594 
1595 // hybrid_ls_rr_sort - Priority function for hybrid scheduler.
1596 struct hybrid_ls_rr_sort : public queue_sort {
1597  enum {
1598  IsBottomUp = true,
1599  HasReadyFilter = false
1600  };
1601 
1602  RegReductionPQBase *SPQ;
1603  hybrid_ls_rr_sort(RegReductionPQBase *spq)
1604  : SPQ(spq) {}
1605 
1606  bool isReady(SUnit *SU, unsigned CurCycle) const;
1607 
1608  bool operator()(SUnit* left, SUnit* right) const;
1609 };
1610 
1611 // ilp_ls_rr_sort - Priority function for ILP (instruction level parallelism)
1612 // scheduler.
1613 struct ilp_ls_rr_sort : public queue_sort {
1614  enum {
1615  IsBottomUp = true,
1616  HasReadyFilter = false
1617  };
1618 
1619  RegReductionPQBase *SPQ;
1620  ilp_ls_rr_sort(RegReductionPQBase *spq)
1621  : SPQ(spq) {}
1622 
1623  bool isReady(SUnit *SU, unsigned CurCycle) const;
1624 
1625  bool operator()(SUnit* left, SUnit* right) const;
1626 };
1627 
1628 class RegReductionPQBase : public SchedulingPriorityQueue {
1629 protected:
1630  std::vector<SUnit*> Queue;
1631  unsigned CurQueueId;
1632  bool TracksRegPressure;
1633  bool SrcOrder;
1634 
1635  // SUnits - The SUnits for the current graph.
1636  std::vector<SUnit> *SUnits;
1637 
1638  MachineFunction &MF;
1639  const TargetInstrInfo *TII;
1640  const TargetRegisterInfo *TRI;
1641  const TargetLowering *TLI;
1642  ScheduleDAGRRList *scheduleDAG;
1643 
1644  // SethiUllmanNumbers - The SethiUllman number for each node.
1645  std::vector<unsigned> SethiUllmanNumbers;
1646 
1647  /// RegPressure - Tracking current reg pressure per register class.
1648  ///
1649  std::vector<unsigned> RegPressure;
1650 
1651  /// RegLimit - Tracking the number of allocatable registers per register
1652  /// class.
1653  std::vector<unsigned> RegLimit;
1654 
1655 public:
1656  RegReductionPQBase(MachineFunction &mf,
1657  bool hasReadyFilter,
1658  bool tracksrp,
1659  bool srcorder,
1660  const TargetInstrInfo *tii,
1661  const TargetRegisterInfo *tri,
1662  const TargetLowering *tli)
1663  : SchedulingPriorityQueue(hasReadyFilter),
1664  CurQueueId(0), TracksRegPressure(tracksrp), SrcOrder(srcorder),
1665  MF(mf), TII(tii), TRI(tri), TLI(tli), scheduleDAG(nullptr) {
1666  if (TracksRegPressure) {
1667  unsigned NumRC = TRI->getNumRegClasses();
1668  RegLimit.resize(NumRC);
1669  RegPressure.resize(NumRC);
1670  std::fill(RegLimit.begin(), RegLimit.end(), 0);
1671  std::fill(RegPressure.begin(), RegPressure.end(), 0);
1672  for (TargetRegisterInfo::regclass_iterator I = TRI->regclass_begin(),
1673  E = TRI->regclass_end(); I != E; ++I)
1674  RegLimit[(*I)->getID()] = tri->getRegPressureLimit(*I, MF);
1675  }
1676  }
1677 
1678  void setScheduleDAG(ScheduleDAGRRList *scheduleDag) {
1679  scheduleDAG = scheduleDag;
1680  }
1681 
1682  ScheduleHazardRecognizer* getHazardRec() {
1683  return scheduleDAG->getHazardRec();
1684  }
1685 
1686  void initNodes(std::vector<SUnit> &sunits) override;
1687 
1688  void addNode(const SUnit *SU) override;
1689 
1690  void updateNode(const SUnit *SU) override;
1691 
1692  void releaseState() override {
1693  SUnits = nullptr;
1694  SethiUllmanNumbers.clear();
1695  std::fill(RegPressure.begin(), RegPressure.end(), 0);
1696  }
1697 
1698  unsigned getNodePriority(const SUnit *SU) const;
1699 
1700  unsigned getNodeOrdering(const SUnit *SU) const {
1701  if (!SU->getNode()) return 0;
1702 
1703  return SU->getNode()->getIROrder();
1704  }
1705 
1706  bool empty() const override { return Queue.empty(); }
1707 
1708  void push(SUnit *U) override {
1709  assert(!U->NodeQueueId && "Node in the queue already");
1710  U->NodeQueueId = ++CurQueueId;
1711  Queue.push_back(U);
1712  }
1713 
1714  void remove(SUnit *SU) override {
1715  assert(!Queue.empty() && "Queue is empty!");
1716  assert(SU->NodeQueueId != 0 && "Not in queue!");
1717  std::vector<SUnit *>::iterator I = std::find(Queue.begin(), Queue.end(),
1718  SU);
1719  if (I != std::prev(Queue.end()))
1720  std::swap(*I, Queue.back());
1721  Queue.pop_back();
1722  SU->NodeQueueId = 0;
1723  }
1724 
1725  bool tracksRegPressure() const override { return TracksRegPressure; }
1726 
1727  void dumpRegPressure() const;
1728 
1729  bool HighRegPressure(const SUnit *SU) const;
1730 
1731  bool MayReduceRegPressure(SUnit *SU) const;
1732 
1733  int RegPressureDiff(SUnit *SU, unsigned &LiveUses) const;
1734 
1735  void scheduledNode(SUnit *SU) override;
1736 
1737  void unscheduledNode(SUnit *SU) override;
1738 
1739 protected:
1740  bool canClobber(const SUnit *SU, const SUnit *Op);
1741  void AddPseudoTwoAddrDeps();
1742  void PrescheduleNodesWithMultipleUses();
1743  void CalculateSethiUllmanNumbers();
1744 };
1745 
1746 template<class SF>
1747 static SUnit *popFromQueueImpl(std::vector<SUnit*> &Q, SF &Picker) {
1748  std::vector<SUnit *>::iterator Best = Q.begin();
1749  for (std::vector<SUnit *>::iterator I = std::next(Q.begin()),
1750  E = Q.end(); I != E; ++I)
1751  if (Picker(*Best, *I))
1752  Best = I;
1753  SUnit *V = *Best;
1754  if (Best != std::prev(Q.end()))
1755  std::swap(*Best, Q.back());
1756  Q.pop_back();
1757  return V;
1758 }
1759 
1760 template<class SF>
1761 SUnit *popFromQueue(std::vector<SUnit*> &Q, SF &Picker, ScheduleDAG *DAG) {
1762 #ifndef NDEBUG
1763  if (DAG->StressSched) {
1764  reverse_sort<SF> RPicker(Picker);
1765  return popFromQueueImpl(Q, RPicker);
1766  }
1767 #endif
1768  (void)DAG;
1769  return popFromQueueImpl(Q, Picker);
1770 }
1771 
1772 template<class SF>
1773 class RegReductionPriorityQueue : public RegReductionPQBase {
1774  SF Picker;
1775 
1776 public:
1777  RegReductionPriorityQueue(MachineFunction &mf,
1778  bool tracksrp,
1779  bool srcorder,
1780  const TargetInstrInfo *tii,
1781  const TargetRegisterInfo *tri,
1782  const TargetLowering *tli)
1783  : RegReductionPQBase(mf, SF::HasReadyFilter, tracksrp, srcorder,
1784  tii, tri, tli),
1785  Picker(this) {}
1786 
1787  bool isBottomUp() const override { return SF::IsBottomUp; }
1788 
1789  bool isReady(SUnit *U) const override {
1790  return Picker.HasReadyFilter && Picker.isReady(U, getCurCycle());
1791  }
1792 
1793  SUnit *pop() override {
1794  if (Queue.empty()) return nullptr;
1795 
1796  SUnit *V = popFromQueue(Queue, Picker, scheduleDAG);
1797  V->NodeQueueId = 0;
1798  return V;
1799  }
1800 
1801 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1802  void dump(ScheduleDAG *DAG) const override {
1803  // Emulate pop() without clobbering NodeQueueIds.
1804  std::vector<SUnit*> DumpQueue = Queue;
1805  SF DumpPicker = Picker;
1806  while (!DumpQueue.empty()) {
1807  SUnit *SU = popFromQueue(DumpQueue, DumpPicker, scheduleDAG);
1808  dbgs() << "Height " << SU->getHeight() << ": ";
1809  SU->dump(DAG);
1810  }
1811  }
1812 #endif
1813 };
1814 
1815 typedef RegReductionPriorityQueue<bu_ls_rr_sort>
1816 BURegReductionPriorityQueue;
1817 
1818 typedef RegReductionPriorityQueue<src_ls_rr_sort>
1819 SrcRegReductionPriorityQueue;
1820 
1821 typedef RegReductionPriorityQueue<hybrid_ls_rr_sort>
1822 HybridBURRPriorityQueue;
1823 
1824 typedef RegReductionPriorityQueue<ilp_ls_rr_sort>
1825 ILPBURRPriorityQueue;
1826 } // end anonymous namespace
1827 
1828 //===----------------------------------------------------------------------===//
1829 // Static Node Priority for Register Pressure Reduction
1830 //===----------------------------------------------------------------------===//
1831 
1832 // Check for special nodes that bypass scheduling heuristics.
1833 // Currently this pushes TokenFactor nodes down, but may be used for other
1834 // pseudo-ops as well.
1835 //
1836 // Return -1 to schedule right above left, 1 for left above right.
1837 // Return 0 if no bias exists.
1838 static int checkSpecialNodes(const SUnit *left, const SUnit *right) {
1839  bool LSchedLow = left->isScheduleLow;
1840  bool RSchedLow = right->isScheduleLow;
1841  if (LSchedLow != RSchedLow)
1842  return LSchedLow < RSchedLow ? 1 : -1;
1843  return 0;
1844 }
1845 
1846 /// CalcNodeSethiUllmanNumber - Compute Sethi Ullman number.
1847 /// Smaller number is the higher priority.
1848 static unsigned
1849 CalcNodeSethiUllmanNumber(const SUnit *SU, std::vector<unsigned> &SUNumbers) {
1850  unsigned &SethiUllmanNumber = SUNumbers[SU->NodeNum];
1851  if (SethiUllmanNumber != 0)
1852  return SethiUllmanNumber;
1853 
1854  unsigned Extra = 0;
1855  for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
1856  I != E; ++I) {
1857  if (I->isCtrl()) continue; // ignore chain preds
1858  SUnit *PredSU = I->getSUnit();
1859  unsigned PredSethiUllman = CalcNodeSethiUllmanNumber(PredSU, SUNumbers);
1860  if (PredSethiUllman > SethiUllmanNumber) {
1861  SethiUllmanNumber = PredSethiUllman;
1862  Extra = 0;
1863  } else if (PredSethiUllman == SethiUllmanNumber)
1864  ++Extra;
1865  }
1866 
1867  SethiUllmanNumber += Extra;
1868 
1869  if (SethiUllmanNumber == 0)
1870  SethiUllmanNumber = 1;
1871 
1872  return SethiUllmanNumber;
1873 }
1874 
1875 /// CalculateSethiUllmanNumbers - Calculate Sethi-Ullman numbers of all
1876 /// scheduling units.
1877 void RegReductionPQBase::CalculateSethiUllmanNumbers() {
1878  SethiUllmanNumbers.assign(SUnits->size(), 0);
1879 
1880  for (unsigned i = 0, e = SUnits->size(); i != e; ++i)
1881  CalcNodeSethiUllmanNumber(&(*SUnits)[i], SethiUllmanNumbers);
1882 }
1883 
1884 void RegReductionPQBase::addNode(const SUnit *SU) {
1885  unsigned SUSize = SethiUllmanNumbers.size();
1886  if (SUnits->size() > SUSize)
1887  SethiUllmanNumbers.resize(SUSize*2, 0);
1888  CalcNodeSethiUllmanNumber(SU, SethiUllmanNumbers);
1889 }
1890 
1891 void RegReductionPQBase::updateNode(const SUnit *SU) {
1892  SethiUllmanNumbers[SU->NodeNum] = 0;
1893  CalcNodeSethiUllmanNumber(SU, SethiUllmanNumbers);
1894 }
1895 
1896 // Lower priority means schedule further down. For bottom-up scheduling, lower
1897 // priority SUs are scheduled before higher priority SUs.
1898 unsigned RegReductionPQBase::getNodePriority(const SUnit *SU) const {
1899  assert(SU->NodeNum < SethiUllmanNumbers.size());
1900  unsigned Opc = SU->getNode() ? SU->getNode()->getOpcode() : 0;
1901  if (Opc == ISD::TokenFactor || Opc == ISD::CopyToReg)
1902  // CopyToReg should be close to its uses to facilitate coalescing and
1903  // avoid spilling.
1904  return 0;
1905  if (Opc == TargetOpcode::EXTRACT_SUBREG ||
1906  Opc == TargetOpcode::SUBREG_TO_REG ||
1908  // EXTRACT_SUBREG, INSERT_SUBREG, and SUBREG_TO_REG nodes should be
1909  // close to their uses to facilitate coalescing.
1910  return 0;
1911  if (SU->NumSuccs == 0 && SU->NumPreds != 0)
1912  // If SU does not have a register use, i.e. it doesn't produce a value
1913  // that would be consumed (e.g. store), then it terminates a chain of
1914  // computation. Give it a large SethiUllman number so it will be
1915  // scheduled right before its predecessors that it doesn't lengthen
1916  // their live ranges.
1917  return 0xffff;
1918  if (SU->NumPreds == 0 && SU->NumSuccs != 0)
1919  // If SU does not have a register def, schedule it close to its uses
1920  // because it does not lengthen any live ranges.
1921  return 0;
1922 #if 1
1923  return SethiUllmanNumbers[SU->NodeNum];
1924 #else
1925  unsigned Priority = SethiUllmanNumbers[SU->NodeNum];
1926  if (SU->isCallOp) {
1927  // FIXME: This assumes all of the defs are used as call operands.
1928  int NP = (int)Priority - SU->getNode()->getNumValues();
1929  return (NP > 0) ? NP : 0;
1930  }
1931  return Priority;
1932 #endif
1933 }
1934 
1935 //===----------------------------------------------------------------------===//
1936 // Register Pressure Tracking
1937 //===----------------------------------------------------------------------===//
1938 
1939 void RegReductionPQBase::dumpRegPressure() const {
1940 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1941  for (TargetRegisterInfo::regclass_iterator I = TRI->regclass_begin(),
1942  E = TRI->regclass_end(); I != E; ++I) {
1943  const TargetRegisterClass *RC = *I;
1944  unsigned Id = RC->getID();
1945  unsigned RP = RegPressure[Id];
1946  if (!RP) continue;
1947  DEBUG(dbgs() << TRI->getRegClassName(RC) << ": " << RP << " / "
1948  << RegLimit[Id] << '\n');
1949  }
1950 #endif
1951 }
1952 
1953 bool RegReductionPQBase::HighRegPressure(const SUnit *SU) const {
1954  if (!TLI)
1955  return false;
1956 
1957  for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end();
1958  I != E; ++I) {
1959  if (I->isCtrl())
1960  continue;
1961  SUnit *PredSU = I->getSUnit();
1962  // NumRegDefsLeft is zero when enough uses of this node have been scheduled
1963  // to cover the number of registers defined (they are all live).
1964  if (PredSU->NumRegDefsLeft == 0) {
1965  continue;
1966  }
1967  for (ScheduleDAGSDNodes::RegDefIter RegDefPos(PredSU, scheduleDAG);
1968  RegDefPos.IsValid(); RegDefPos.Advance()) {
1969  unsigned RCId, Cost;
1970  GetCostForDef(RegDefPos, TLI, TII, TRI, RCId, Cost, MF);
1971 
1972  if ((RegPressure[RCId] + Cost) >= RegLimit[RCId])
1973  return true;
1974  }
1975  }
1976  return false;
1977 }
1978 
1979 bool RegReductionPQBase::MayReduceRegPressure(SUnit *SU) const {
1980  const SDNode *N = SU->getNode();
1981 
1982  if (!N->isMachineOpcode() || !SU->NumSuccs)
1983  return false;
1984 
1985  unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs();
1986  for (unsigned i = 0; i != NumDefs; ++i) {
1987  MVT VT = N->getSimpleValueType(i);
1988  if (!N->hasAnyUseOfValue(i))
1989  continue;
1990  unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
1991  if (RegPressure[RCId] >= RegLimit[RCId])
1992  return true;
1993  }
1994  return false;
1995 }
1996 
1997 // Compute the register pressure contribution by this instruction by count up
1998 // for uses that are not live and down for defs. Only count register classes
1999 // that are already under high pressure. As a side effect, compute the number of
2000 // uses of registers that are already live.
2001 //
2002 // FIXME: This encompasses the logic in HighRegPressure and MayReduceRegPressure
2003 // so could probably be factored.
2004 int RegReductionPQBase::RegPressureDiff(SUnit *SU, unsigned &LiveUses) const {
2005  LiveUses = 0;
2006  int PDiff = 0;
2007  for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end();
2008  I != E; ++I) {
2009  if (I->isCtrl())
2010  continue;
2011  SUnit *PredSU = I->getSUnit();
2012  // NumRegDefsLeft is zero when enough uses of this node have been scheduled
2013  // to cover the number of registers defined (they are all live).
2014  if (PredSU->NumRegDefsLeft == 0) {
2015  if (PredSU->getNode()->isMachineOpcode())
2016  ++LiveUses;
2017  continue;
2018  }
2019  for (ScheduleDAGSDNodes::RegDefIter RegDefPos(PredSU, scheduleDAG);
2020  RegDefPos.IsValid(); RegDefPos.Advance()) {
2021  MVT VT = RegDefPos.GetValue();
2022  unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
2023  if (RegPressure[RCId] >= RegLimit[RCId])
2024  ++PDiff;
2025  }
2026  }
2027  const SDNode *N = SU->getNode();
2028 
2029  if (!N || !N->isMachineOpcode() || !SU->NumSuccs)
2030  return PDiff;
2031 
2032  unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs();
2033  for (unsigned i = 0; i != NumDefs; ++i) {
2034  MVT VT = N->getSimpleValueType(i);
2035  if (!N->hasAnyUseOfValue(i))
2036  continue;
2037  unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
2038  if (RegPressure[RCId] >= RegLimit[RCId])
2039  --PDiff;
2040  }
2041  return PDiff;
2042 }
2043 
2044 void RegReductionPQBase::scheduledNode(SUnit *SU) {
2045  if (!TracksRegPressure)
2046  return;
2047 
2048  if (!SU->getNode())
2049  return;
2050 
2051  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
2052  I != E; ++I) {
2053  if (I->isCtrl())
2054  continue;
2055  SUnit *PredSU = I->getSUnit();
2056  // NumRegDefsLeft is zero when enough uses of this node have been scheduled
2057  // to cover the number of registers defined (they are all live).
2058  if (PredSU->NumRegDefsLeft == 0) {
2059  continue;
2060  }
2061  // FIXME: The ScheduleDAG currently loses information about which of a
2062  // node's values is consumed by each dependence. Consequently, if the node
2063  // defines multiple register classes, we don't know which to pressurize
2064  // here. Instead the following loop consumes the register defs in an
2065  // arbitrary order. At least it handles the common case of clustered loads
2066  // to the same class. For precise liveness, each SDep needs to indicate the
2067  // result number. But that tightly couples the ScheduleDAG with the
2068  // SelectionDAG making updates tricky. A simpler hack would be to attach a
2069  // value type or register class to SDep.
2070  //
2071  // The most important aspect of register tracking is balancing the increase
2072  // here with the reduction further below. Note that this SU may use multiple
2073  // defs in PredSU. The can't be determined here, but we've already
2074  // compensated by reducing NumRegDefsLeft in PredSU during
2075  // ScheduleDAGSDNodes::AddSchedEdges.
2076  --PredSU->NumRegDefsLeft;
2077  unsigned SkipRegDefs = PredSU->NumRegDefsLeft;
2078  for (ScheduleDAGSDNodes::RegDefIter RegDefPos(PredSU, scheduleDAG);
2079  RegDefPos.IsValid(); RegDefPos.Advance(), --SkipRegDefs) {
2080  if (SkipRegDefs)
2081  continue;
2082 
2083  unsigned RCId, Cost;
2084  GetCostForDef(RegDefPos, TLI, TII, TRI, RCId, Cost, MF);
2085  RegPressure[RCId] += Cost;
2086  break;
2087  }
2088  }
2089 
2090  // We should have this assert, but there may be dead SDNodes that never
2091  // materialize as SUnits, so they don't appear to generate liveness.
2092  //assert(SU->NumRegDefsLeft == 0 && "not all regdefs have scheduled uses");
2093  int SkipRegDefs = (int)SU->NumRegDefsLeft;
2094  for (ScheduleDAGSDNodes::RegDefIter RegDefPos(SU, scheduleDAG);
2095  RegDefPos.IsValid(); RegDefPos.Advance(), --SkipRegDefs) {
2096  if (SkipRegDefs > 0)
2097  continue;
2098  unsigned RCId, Cost;
2099  GetCostForDef(RegDefPos, TLI, TII, TRI, RCId, Cost, MF);
2100  if (RegPressure[RCId] < Cost) {
2101  // Register pressure tracking is imprecise. This can happen. But we try
2102  // hard not to let it happen because it likely results in poor scheduling.
2103  DEBUG(dbgs() << " SU(" << SU->NodeNum << ") has too many regdefs\n");
2104  RegPressure[RCId] = 0;
2105  }
2106  else {
2107  RegPressure[RCId] -= Cost;
2108  }
2109  }
2110  dumpRegPressure();
2111 }
2112 
2113 void RegReductionPQBase::unscheduledNode(SUnit *SU) {
2114  if (!TracksRegPressure)
2115  return;
2116 
2117  const SDNode *N = SU->getNode();
2118  if (!N) return;
2119 
2120  if (!N->isMachineOpcode()) {
2121  if (N->getOpcode() != ISD::CopyToReg)
2122  return;
2123  } else {
2124  unsigned Opc = N->getMachineOpcode();
2125  if (Opc == TargetOpcode::EXTRACT_SUBREG ||
2126  Opc == TargetOpcode::INSERT_SUBREG ||
2127  Opc == TargetOpcode::SUBREG_TO_REG ||
2128  Opc == TargetOpcode::REG_SEQUENCE ||
2130  return;
2131  }
2132 
2133  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
2134  I != E; ++I) {
2135  if (I->isCtrl())
2136  continue;
2137  SUnit *PredSU = I->getSUnit();
2138  // NumSuccsLeft counts all deps. Don't compare it with NumSuccs which only
2139  // counts data deps.
2140  if (PredSU->NumSuccsLeft != PredSU->Succs.size())
2141  continue;
2142  const SDNode *PN = PredSU->getNode();
2143  if (!PN->isMachineOpcode()) {
2144  if (PN->getOpcode() == ISD::CopyFromReg) {
2145  MVT VT = PN->getSimpleValueType(0);
2146  unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
2147  RegPressure[RCId] += TLI->getRepRegClassCostFor(VT);
2148  }
2149  continue;
2150  }
2151  unsigned POpc = PN->getMachineOpcode();
2152  if (POpc == TargetOpcode::IMPLICIT_DEF)
2153  continue;
2154  if (POpc == TargetOpcode::EXTRACT_SUBREG ||
2155  POpc == TargetOpcode::INSERT_SUBREG ||
2156  POpc == TargetOpcode::SUBREG_TO_REG) {
2157  MVT VT = PN->getSimpleValueType(0);
2158  unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
2159  RegPressure[RCId] += TLI->getRepRegClassCostFor(VT);
2160  continue;
2161  }
2162  unsigned NumDefs = TII->get(PN->getMachineOpcode()).getNumDefs();
2163  for (unsigned i = 0; i != NumDefs; ++i) {
2164  MVT VT = PN->getSimpleValueType(i);
2165  if (!PN->hasAnyUseOfValue(i))
2166  continue;
2167  unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
2168  if (RegPressure[RCId] < TLI->getRepRegClassCostFor(VT))
2169  // Register pressure tracking is imprecise. This can happen.
2170  RegPressure[RCId] = 0;
2171  else
2172  RegPressure[RCId] -= TLI->getRepRegClassCostFor(VT);
2173  }
2174  }
2175 
2176  // Check for isMachineOpcode() as PrescheduleNodesWithMultipleUses()
2177  // may transfer data dependencies to CopyToReg.
2178  if (SU->NumSuccs && N->isMachineOpcode()) {
2179  unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs();
2180  for (unsigned i = NumDefs, e = N->getNumValues(); i != e; ++i) {
2181  MVT VT = N->getSimpleValueType(i);
2182  if (VT == MVT::Glue || VT == MVT::Other)
2183  continue;
2184  if (!N->hasAnyUseOfValue(i))
2185  continue;
2186  unsigned RCId = TLI->getRepRegClassFor(VT)->getID();
2187  RegPressure[RCId] += TLI->getRepRegClassCostFor(VT);
2188  }
2189  }
2190 
2191  dumpRegPressure();
2192 }
2193 
2194 //===----------------------------------------------------------------------===//
2195 // Dynamic Node Priority for Register Pressure Reduction
2196 //===----------------------------------------------------------------------===//
2197 
2198 /// closestSucc - Returns the scheduled cycle of the successor which is
2199 /// closest to the current cycle.
2200 static unsigned closestSucc(const SUnit *SU) {
2201  unsigned MaxHeight = 0;
2202  for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
2203  I != E; ++I) {
2204  if (I->isCtrl()) continue; // ignore chain succs
2205  unsigned Height = I->getSUnit()->getHeight();
2206  // If there are bunch of CopyToRegs stacked up, they should be considered
2207  // to be at the same position.
2208  if (I->getSUnit()->getNode() &&
2209  I->getSUnit()->getNode()->getOpcode() == ISD::CopyToReg)
2210  Height = closestSucc(I->getSUnit())+1;
2211  if (Height > MaxHeight)
2212  MaxHeight = Height;
2213  }
2214  return MaxHeight;
2215 }
2216 
2217 /// calcMaxScratches - Returns an cost estimate of the worse case requirement
2218 /// for scratch registers, i.e. number of data dependencies.
2219 static unsigned calcMaxScratches(const SUnit *SU) {
2220  unsigned Scratches = 0;
2221  for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
2222  I != E; ++I) {
2223  if (I->isCtrl()) continue; // ignore chain preds
2224  Scratches++;
2225  }
2226  return Scratches;
2227 }
2228 
2229 /// hasOnlyLiveInOpers - Return true if SU has only value predecessors that are
2230 /// CopyFromReg from a virtual register.
2231 static bool hasOnlyLiveInOpers(const SUnit *SU) {
2232  bool RetVal = false;
2233  for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
2234  I != E; ++I) {
2235  if (I->isCtrl()) continue;
2236  const SUnit *PredSU = I->getSUnit();
2237  if (PredSU->getNode() &&
2238  PredSU->getNode()->getOpcode() == ISD::CopyFromReg) {
2239  unsigned Reg =
2240  cast<RegisterSDNode>(PredSU->getNode()->getOperand(1))->getReg();
2242  RetVal = true;
2243  continue;
2244  }
2245  }
2246  return false;
2247  }
2248  return RetVal;
2249 }
2250 
2251 /// hasOnlyLiveOutUses - Return true if SU has only value successors that are
2252 /// CopyToReg to a virtual register. This SU def is probably a liveout and
2253 /// it has no other use. It should be scheduled closer to the terminator.
2254 static bool hasOnlyLiveOutUses(const SUnit *SU) {
2255  bool RetVal = false;
2256  for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
2257  I != E; ++I) {
2258  if (I->isCtrl()) continue;
2259  const SUnit *SuccSU = I->getSUnit();
2260  if (SuccSU->getNode() && SuccSU->getNode()->getOpcode() == ISD::CopyToReg) {
2261  unsigned Reg =
2262  cast<RegisterSDNode>(SuccSU->getNode()->getOperand(1))->getReg();
2264  RetVal = true;
2265  continue;
2266  }
2267  }
2268  return false;
2269  }
2270  return RetVal;
2271 }
2272 
2273 // Set isVRegCycle for a node with only live in opers and live out uses. Also
2274 // set isVRegCycle for its CopyFromReg operands.
2275 //
2276 // This is only relevant for single-block loops, in which case the VRegCycle
2277 // node is likely an induction variable in which the operand and target virtual
2278 // registers should be coalesced (e.g. pre/post increment values). Setting the
2279 // isVRegCycle flag helps the scheduler prioritize other uses of the same
2280 // CopyFromReg so that this node becomes the virtual register "kill". This
2281 // avoids interference between the values live in and out of the block and
2282 // eliminates a copy inside the loop.
2283 static void initVRegCycle(SUnit *SU) {
2285  return;
2286 
2287  if (!hasOnlyLiveInOpers(SU) || !hasOnlyLiveOutUses(SU))
2288  return;
2289 
2290  DEBUG(dbgs() << "VRegCycle: SU(" << SU->NodeNum << ")\n");
2291 
2292  SU->isVRegCycle = true;
2293 
2294  for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
2295  I != E; ++I) {
2296  if (I->isCtrl()) continue;
2297  I->getSUnit()->isVRegCycle = true;
2298  }
2299 }
2300 
2301 // After scheduling the definition of a VRegCycle, clear the isVRegCycle flag of
2302 // CopyFromReg operands. We should no longer penalize other uses of this VReg.
2303 static void resetVRegCycle(SUnit *SU) {
2304  if (!SU->isVRegCycle)
2305  return;
2306 
2307  for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end();
2308  I != E; ++I) {
2309  if (I->isCtrl()) continue; // ignore chain preds
2310  SUnit *PredSU = I->getSUnit();
2311  if (PredSU->isVRegCycle) {
2312  assert(PredSU->getNode()->getOpcode() == ISD::CopyFromReg &&
2313  "VRegCycle def must be CopyFromReg");
2314  I->getSUnit()->isVRegCycle = 0;
2315  }
2316  }
2317 }
2318 
2319 // Return true if this SUnit uses a CopyFromReg node marked as a VRegCycle. This
2320 // means a node that defines the VRegCycle has not been scheduled yet.
2321 static bool hasVRegCycleUse(const SUnit *SU) {
2322  // If this SU also defines the VReg, don't hoist it as a "use".
2323  if (SU->isVRegCycle)
2324  return false;
2325 
2326  for (SUnit::const_pred_iterator I = SU->Preds.begin(),E = SU->Preds.end();
2327  I != E; ++I) {
2328  if (I->isCtrl()) continue; // ignore chain preds
2329  if (I->getSUnit()->isVRegCycle &&
2330  I->getSUnit()->getNode()->getOpcode() == ISD::CopyFromReg) {
2331  DEBUG(dbgs() << " VReg cycle use: SU (" << SU->NodeNum << ")\n");
2332  return true;
2333  }
2334  }
2335  return false;
2336 }
2337 
2338 // Check for either a dependence (latency) or resource (hazard) stall.
2339 //
2340 // Note: The ScheduleHazardRecognizer interface requires a non-const SU.
2341 static bool BUHasStall(SUnit *SU, int Height, RegReductionPQBase *SPQ) {
2342  if ((int)SPQ->getCurCycle() < Height) return true;
2343  if (SPQ->getHazardRec()->getHazardType(SU, 0)
2345  return true;
2346  return false;
2347 }
2348 
2349 // Return -1 if left has higher priority, 1 if right has higher priority.
2350 // Return 0 if latency-based priority is equivalent.
2351 static int BUCompareLatency(SUnit *left, SUnit *right, bool checkPref,
2352  RegReductionPQBase *SPQ) {
2353  // Scheduling an instruction that uses a VReg whose postincrement has not yet
2354  // been scheduled will induce a copy. Model this as an extra cycle of latency.
2355  int LPenalty = hasVRegCycleUse(left) ? 1 : 0;
2356  int RPenalty = hasVRegCycleUse(right) ? 1 : 0;
2357  int LHeight = (int)left->getHeight() + LPenalty;
2358  int RHeight = (int)right->getHeight() + RPenalty;
2359 
2360  bool LStall = (!checkPref || left->SchedulingPref == Sched::ILP) &&
2361  BUHasStall(left, LHeight, SPQ);
2362  bool RStall = (!checkPref || right->SchedulingPref == Sched::ILP) &&
2363  BUHasStall(right, RHeight, SPQ);
2364 
2365  // If scheduling one of the node will cause a pipeline stall, delay it.
2366  // If scheduling either one of the node will cause a pipeline stall, sort
2367  // them according to their height.
2368  if (LStall) {
2369  if (!RStall)
2370  return 1;
2371  if (LHeight != RHeight)
2372  return LHeight > RHeight ? 1 : -1;
2373  } else if (RStall)
2374  return -1;
2375 
2376  // If either node is scheduling for latency, sort them by height/depth
2377  // and latency.
2378  if (!checkPref || (left->SchedulingPref == Sched::ILP ||
2379  right->SchedulingPref == Sched::ILP)) {
2380  // If neither instruction stalls (!LStall && !RStall) and HazardRecognizer
2381  // is enabled, grouping instructions by cycle, then its height is already
2382  // covered so only its depth matters. We also reach this point if both stall
2383  // but have the same height.
2384  if (!SPQ->getHazardRec()->isEnabled()) {
2385  if (LHeight != RHeight)
2386  return LHeight > RHeight ? 1 : -1;
2387  }
2388  int LDepth = left->getDepth() - LPenalty;
2389  int RDepth = right->getDepth() - RPenalty;
2390  if (LDepth != RDepth) {
2391  DEBUG(dbgs() << " Comparing latency of SU (" << left->NodeNum
2392  << ") depth " << LDepth << " vs SU (" << right->NodeNum
2393  << ") depth " << RDepth << "\n");
2394  return LDepth < RDepth ? 1 : -1;
2395  }
2396  if (left->Latency != right->Latency)
2397  return left->Latency > right->Latency ? 1 : -1;
2398  }
2399  return 0;
2400 }
2401 
2402 static bool BURRSort(SUnit *left, SUnit *right, RegReductionPQBase *SPQ) {
2403  // Schedule physical register definitions close to their use. This is
2404  // motivated by microarchitectures that can fuse cmp+jump macro-ops. But as
2405  // long as shortening physreg live ranges is generally good, we can defer
2406  // creating a subtarget hook.
2407  if (!DisableSchedPhysRegJoin) {
2408  bool LHasPhysReg = left->hasPhysRegDefs;
2409  bool RHasPhysReg = right->hasPhysRegDefs;
2410  if (LHasPhysReg != RHasPhysReg) {
2411  #ifndef NDEBUG
2412  static const char *const PhysRegMsg[] = { " has no physreg",
2413  " defines a physreg" };
2414  #endif
2415  DEBUG(dbgs() << " SU (" << left->NodeNum << ") "
2416  << PhysRegMsg[LHasPhysReg] << " SU(" << right->NodeNum << ") "
2417  << PhysRegMsg[RHasPhysReg] << "\n");
2418  return LHasPhysReg < RHasPhysReg;
2419  }
2420  }
2421 
2422  // Prioritize by Sethi-Ulmann number and push CopyToReg nodes down.
2423  unsigned LPriority = SPQ->getNodePriority(left);
2424  unsigned RPriority = SPQ->getNodePriority(right);
2425 
2426  // Be really careful about hoisting call operands above previous calls.
2427  // Only allows it if it would reduce register pressure.
2428  if (left->isCall && right->isCallOp) {
2429  unsigned RNumVals = right->getNode()->getNumValues();
2430  RPriority = (RPriority > RNumVals) ? (RPriority - RNumVals) : 0;
2431  }
2432  if (right->isCall && left->isCallOp) {
2433  unsigned LNumVals = left->getNode()->getNumValues();
2434  LPriority = (LPriority > LNumVals) ? (LPriority - LNumVals) : 0;
2435  }
2436 
2437  if (LPriority != RPriority)
2438  return LPriority > RPriority;
2439 
2440  // One or both of the nodes are calls and their sethi-ullman numbers are the
2441  // same, then keep source order.
2442  if (left->isCall || right->isCall) {
2443  unsigned LOrder = SPQ->getNodeOrdering(left);
2444  unsigned ROrder = SPQ->getNodeOrdering(right);
2445 
2446  // Prefer an ordering where the lower the non-zero order number, the higher
2447  // the preference.
2448  if ((LOrder || ROrder) && LOrder != ROrder)
2449  return LOrder != 0 && (LOrder < ROrder || ROrder == 0);
2450  }
2451 
2452  // Try schedule def + use closer when Sethi-Ullman numbers are the same.
2453  // e.g.
2454  // t1 = op t2, c1
2455  // t3 = op t4, c2
2456  //
2457  // and the following instructions are both ready.
2458  // t2 = op c3
2459  // t4 = op c4
2460  //
2461  // Then schedule t2 = op first.
2462  // i.e.
2463  // t4 = op c4
2464  // t2 = op c3
2465  // t1 = op t2, c1
2466  // t3 = op t4, c2
2467  //
2468  // This creates more short live intervals.
2469  unsigned LDist = closestSucc(left);
2470  unsigned RDist = closestSucc(right);
2471  if (LDist != RDist)
2472  return LDist < RDist;
2473 
2474  // How many registers becomes live when the node is scheduled.
2475  unsigned LScratch = calcMaxScratches(left);
2476  unsigned RScratch = calcMaxScratches(right);
2477  if (LScratch != RScratch)
2478  return LScratch > RScratch;
2479 
2480  // Comparing latency against a call makes little sense unless the node
2481  // is register pressure-neutral.
2482  if ((left->isCall && RPriority > 0) || (right->isCall && LPriority > 0))
2483  return (left->NodeQueueId > right->NodeQueueId);
2484 
2485  // Do not compare latencies when one or both of the nodes are calls.
2486  if (!DisableSchedCycles &&
2487  !(left->isCall || right->isCall)) {
2488  int result = BUCompareLatency(left, right, false /*checkPref*/, SPQ);
2489  if (result != 0)
2490  return result > 0;
2491  }
2492  else {
2493  if (left->getHeight() != right->getHeight())
2494  return left->getHeight() > right->getHeight();
2495 
2496  if (left->getDepth() != right->getDepth())
2497  return left->getDepth() < right->getDepth();
2498  }
2499 
2500  assert(left->NodeQueueId && right->NodeQueueId &&
2501  "NodeQueueId cannot be zero");
2502  return (left->NodeQueueId > right->NodeQueueId);
2503 }
2504 
2505 // Bottom up
2506 bool bu_ls_rr_sort::operator()(SUnit *left, SUnit *right) const {
2507  if (int res = checkSpecialNodes(left, right))
2508  return res > 0;
2509 
2510  return BURRSort(left, right, SPQ);
2511 }
2512 
2513 // Source order, otherwise bottom up.
2514 bool src_ls_rr_sort::operator()(SUnit *left, SUnit *right) const {
2515  if (int res = checkSpecialNodes(left, right))
2516  return res > 0;
2517 
2518  unsigned LOrder = SPQ->getNodeOrdering(left);
2519  unsigned ROrder = SPQ->getNodeOrdering(right);
2520 
2521  // Prefer an ordering where the lower the non-zero order number, the higher
2522  // the preference.
2523  if ((LOrder || ROrder) && LOrder != ROrder)
2524  return LOrder != 0 && (LOrder < ROrder || ROrder == 0);
2525 
2526  return BURRSort(left, right, SPQ);
2527 }
2528 
2529 // If the time between now and when the instruction will be ready can cover
2530 // the spill code, then avoid adding it to the ready queue. This gives long
2531 // stalls highest priority and allows hoisting across calls. It should also
2532 // speed up processing the available queue.
2533 bool hybrid_ls_rr_sort::isReady(SUnit *SU, unsigned CurCycle) const {
2534  static const unsigned ReadyDelay = 3;
2535 
2536  if (SPQ->MayReduceRegPressure(SU)) return true;
2537 
2538  if (SU->getHeight() > (CurCycle + ReadyDelay)) return false;
2539 
2540  if (SPQ->getHazardRec()->getHazardType(SU, -ReadyDelay)
2542  return false;
2543 
2544  return true;
2545 }
2546 
2547 // Return true if right should be scheduled with higher priority than left.
2548 bool hybrid_ls_rr_sort::operator()(SUnit *left, SUnit *right) const {
2549  if (int res = checkSpecialNodes(left, right))
2550  return res > 0;
2551 
2552  if (left->isCall || right->isCall)
2553  // No way to compute latency of calls.
2554  return BURRSort(left, right, SPQ);
2555 
2556  bool LHigh = SPQ->HighRegPressure(left);
2557  bool RHigh = SPQ->HighRegPressure(right);
2558  // Avoid causing spills. If register pressure is high, schedule for
2559  // register pressure reduction.
2560  if (LHigh && !RHigh) {
2561  DEBUG(dbgs() << " pressure SU(" << left->NodeNum << ") > SU("
2562  << right->NodeNum << ")\n");
2563  return true;
2564  }
2565  else if (!LHigh && RHigh) {
2566  DEBUG(dbgs() << " pressure SU(" << right->NodeNum << ") > SU("
2567  << left->NodeNum << ")\n");
2568  return false;
2569  }
2570  if (!LHigh && !RHigh) {
2571  int result = BUCompareLatency(left, right, true /*checkPref*/, SPQ);
2572  if (result != 0)
2573  return result > 0;
2574  }
2575  return BURRSort(left, right, SPQ);
2576 }
2577 
2578 // Schedule as many instructions in each cycle as possible. So don't make an
2579 // instruction available unless it is ready in the current cycle.
2580 bool ilp_ls_rr_sort::isReady(SUnit *SU, unsigned CurCycle) const {
2581  if (SU->getHeight() > CurCycle) return false;
2582 
2583  if (SPQ->getHazardRec()->getHazardType(SU, 0)
2585  return false;
2586 
2587  return true;
2588 }
2589 
2590 static bool canEnableCoalescing(SUnit *SU) {
2591  unsigned Opc = SU->getNode() ? SU->getNode()->getOpcode() : 0;
2592  if (Opc == ISD::TokenFactor || Opc == ISD::CopyToReg)
2593  // CopyToReg should be close to its uses to facilitate coalescing and
2594  // avoid spilling.
2595  return true;
2596 
2597  if (Opc == TargetOpcode::EXTRACT_SUBREG ||
2598  Opc == TargetOpcode::SUBREG_TO_REG ||
2600  // EXTRACT_SUBREG, INSERT_SUBREG, and SUBREG_TO_REG nodes should be
2601  // close to their uses to facilitate coalescing.
2602  return true;
2603 
2604  if (SU->NumPreds == 0 && SU->NumSuccs != 0)
2605  // If SU does not have a register def, schedule it close to its uses
2606  // because it does not lengthen any live ranges.
2607  return true;
2608 
2609  return false;
2610 }
2611 
2612 // list-ilp is currently an experimental scheduler that allows various
2613 // heuristics to be enabled prior to the normal register reduction logic.
2614 bool ilp_ls_rr_sort::operator()(SUnit *left, SUnit *right) const {
2615  if (int res = checkSpecialNodes(left, right))
2616  return res > 0;
2617 
2618  if (left->isCall || right->isCall)
2619  // No way to compute latency of calls.
2620  return BURRSort(left, right, SPQ);
2621 
2622  unsigned LLiveUses = 0, RLiveUses = 0;
2623  int LPDiff = 0, RPDiff = 0;
2625  LPDiff = SPQ->RegPressureDiff(left, LLiveUses);
2626  RPDiff = SPQ->RegPressureDiff(right, RLiveUses);
2627  }
2628  if (!DisableSchedRegPressure && LPDiff != RPDiff) {
2629  DEBUG(dbgs() << "RegPressureDiff SU(" << left->NodeNum << "): " << LPDiff
2630  << " != SU(" << right->NodeNum << "): " << RPDiff << "\n");
2631  return LPDiff > RPDiff;
2632  }
2633 
2634  if (!DisableSchedRegPressure && (LPDiff > 0 || RPDiff > 0)) {
2635  bool LReduce = canEnableCoalescing(left);
2636  bool RReduce = canEnableCoalescing(right);
2637  if (LReduce && !RReduce) return false;
2638  if (RReduce && !LReduce) return true;
2639  }
2640 
2641  if (!DisableSchedLiveUses && (LLiveUses != RLiveUses)) {
2642  DEBUG(dbgs() << "Live uses SU(" << left->NodeNum << "): " << LLiveUses
2643  << " != SU(" << right->NodeNum << "): " << RLiveUses << "\n");
2644  return LLiveUses < RLiveUses;
2645  }
2646 
2647  if (!DisableSchedStalls) {
2648  bool LStall = BUHasStall(left, left->getHeight(), SPQ);
2649  bool RStall = BUHasStall(right, right->getHeight(), SPQ);
2650  if (LStall != RStall)
2651  return left->getHeight() > right->getHeight();
2652  }
2653 
2654  if (!DisableSchedCriticalPath) {
2655  int spread = (int)left->getDepth() - (int)right->getDepth();
2656  if (std::abs(spread) > MaxReorderWindow) {
2657  DEBUG(dbgs() << "Depth of SU(" << left->NodeNum << "): "
2658  << left->getDepth() << " != SU(" << right->NodeNum << "): "
2659  << right->getDepth() << "\n");
2660  return left->getDepth() < right->getDepth();
2661  }
2662  }
2663 
2664  if (!DisableSchedHeight && left->getHeight() != right->getHeight()) {
2665  int spread = (int)left->getHeight() - (int)right->getHeight();
2666  if (std::abs(spread) > MaxReorderWindow)
2667  return left->getHeight() > right->getHeight();
2668  }
2669 
2670  return BURRSort(left, right, SPQ);
2671 }
2672 
2673 void RegReductionPQBase::initNodes(std::vector<SUnit> &sunits) {
2674  SUnits = &sunits;
2675  // Add pseudo dependency edges for two-address nodes.
2676  if (!Disable2AddrHack)
2677  AddPseudoTwoAddrDeps();
2678  // Reroute edges to nodes with multiple uses.
2679  if (!TracksRegPressure && !SrcOrder)
2680  PrescheduleNodesWithMultipleUses();
2681  // Calculate node priorities.
2682  CalculateSethiUllmanNumbers();
2683 
2684  // For single block loops, mark nodes that look like canonical IV increments.
2685  if (scheduleDAG->BB->isSuccessor(scheduleDAG->BB)) {
2686  for (unsigned i = 0, e = sunits.size(); i != e; ++i) {
2687  initVRegCycle(&sunits[i]);
2688  }
2689  }
2690 }
2691 
2692 //===----------------------------------------------------------------------===//
2693 // Preschedule for Register Pressure
2694 //===----------------------------------------------------------------------===//
2695 
2696 bool RegReductionPQBase::canClobber(const SUnit *SU, const SUnit *Op) {
2697  if (SU->isTwoAddress) {
2698  unsigned Opc = SU->getNode()->getMachineOpcode();
2699  const MCInstrDesc &MCID = TII->get(Opc);
2700  unsigned NumRes = MCID.getNumDefs();
2701  unsigned NumOps = MCID.getNumOperands() - NumRes;
2702  for (unsigned i = 0; i != NumOps; ++i) {
2703  if (MCID.getOperandConstraint(i+NumRes, MCOI::TIED_TO) != -1) {
2704  SDNode *DU = SU->getNode()->getOperand(i).getNode();
2705  if (DU->getNodeId() != -1 &&
2706  Op->OrigNode == &(*SUnits)[DU->getNodeId()])
2707  return true;
2708  }
2709  }
2710  }
2711  return false;
2712 }
2713 
2714 /// canClobberReachingPhysRegUse - True if SU would clobber one of it's
2715 /// successor's explicit physregs whose definition can reach DepSU.
2716 /// i.e. DepSU should not be scheduled above SU.
2717 static bool canClobberReachingPhysRegUse(const SUnit *DepSU, const SUnit *SU,
2718  ScheduleDAGRRList *scheduleDAG,
2719  const TargetInstrInfo *TII,
2720  const TargetRegisterInfo *TRI) {
2721  const uint16_t *ImpDefs
2722  = TII->get(SU->getNode()->getMachineOpcode()).getImplicitDefs();
2723  const uint32_t *RegMask = getNodeRegMask(SU->getNode());
2724  if(!ImpDefs && !RegMask)
2725  return false;
2726 
2727  for (SUnit::const_succ_iterator SI = SU->Succs.begin(), SE = SU->Succs.end();
2728  SI != SE; ++SI) {
2729  SUnit *SuccSU = SI->getSUnit();
2730  for (SUnit::const_pred_iterator PI = SuccSU->Preds.begin(),
2731  PE = SuccSU->Preds.end(); PI != PE; ++PI) {
2732  if (!PI->isAssignedRegDep())
2733  continue;
2734 
2735  if (RegMask && MachineOperand::clobbersPhysReg(RegMask, PI->getReg()) &&
2736  scheduleDAG->IsReachable(DepSU, PI->getSUnit()))
2737  return true;
2738 
2739  if (ImpDefs)
2740  for (const uint16_t *ImpDef = ImpDefs; *ImpDef; ++ImpDef)
2741  // Return true if SU clobbers this physical register use and the
2742  // definition of the register reaches from DepSU. IsReachable queries
2743  // a topological forward sort of the DAG (following the successors).
2744  if (TRI->regsOverlap(*ImpDef, PI->getReg()) &&
2745  scheduleDAG->IsReachable(DepSU, PI->getSUnit()))
2746  return true;
2747  }
2748  }
2749  return false;
2750 }
2751 
2752 /// canClobberPhysRegDefs - True if SU would clobber one of SuccSU's
2753 /// physical register defs.
2754 static bool canClobberPhysRegDefs(const SUnit *SuccSU, const SUnit *SU,
2755  const TargetInstrInfo *TII,
2756  const TargetRegisterInfo *TRI) {
2757  SDNode *N = SuccSU->getNode();
2758  unsigned NumDefs = TII->get(N->getMachineOpcode()).getNumDefs();
2759  const uint16_t *ImpDefs = TII->get(N->getMachineOpcode()).getImplicitDefs();
2760  assert(ImpDefs && "Caller should check hasPhysRegDefs");
2761  for (const SDNode *SUNode = SU->getNode(); SUNode;
2762  SUNode = SUNode->getGluedNode()) {
2763  if (!SUNode->isMachineOpcode())
2764  continue;
2765  const uint16_t *SUImpDefs =
2766  TII->get(SUNode->getMachineOpcode()).getImplicitDefs();
2767  const uint32_t *SURegMask = getNodeRegMask(SUNode);
2768  if (!SUImpDefs && !SURegMask)
2769  continue;
2770  for (unsigned i = NumDefs, e = N->getNumValues(); i != e; ++i) {
2771  MVT VT = N->getSimpleValueType(i);
2772  if (VT == MVT::Glue || VT == MVT::Other)
2773  continue;
2774  if (!N->hasAnyUseOfValue(i))
2775  continue;
2776  unsigned Reg = ImpDefs[i - NumDefs];
2777  if (SURegMask && MachineOperand::clobbersPhysReg(SURegMask, Reg))
2778  return true;
2779  if (!SUImpDefs)
2780  continue;
2781  for (;*SUImpDefs; ++SUImpDefs) {
2782  unsigned SUReg = *SUImpDefs;
2783  if (TRI->regsOverlap(Reg, SUReg))
2784  return true;
2785  }
2786  }
2787  }
2788  return false;
2789 }
2790 
2791 /// PrescheduleNodesWithMultipleUses - Nodes with multiple uses
2792 /// are not handled well by the general register pressure reduction
2793 /// heuristics. When presented with code like this:
2794 ///
2795 /// N
2796 /// / |
2797 /// / |
2798 /// U store
2799 /// |
2800 /// ...
2801 ///
2802 /// the heuristics tend to push the store up, but since the
2803 /// operand of the store has another use (U), this would increase
2804 /// the length of that other use (the U->N edge).
2805 ///
2806 /// This function transforms code like the above to route U's
2807 /// dependence through the store when possible, like this:
2808 ///
2809 /// N
2810 /// ||
2811 /// ||
2812 /// store
2813 /// |
2814 /// U
2815 /// |
2816 /// ...
2817 ///
2818 /// This results in the store being scheduled immediately
2819 /// after N, which shortens the U->N live range, reducing
2820 /// register pressure.
2821 ///
2822 void RegReductionPQBase::PrescheduleNodesWithMultipleUses() {
2823  // Visit all the nodes in topological order, working top-down.
2824  for (unsigned i = 0, e = SUnits->size(); i != e; ++i) {
2825  SUnit *SU = &(*SUnits)[i];
2826  // For now, only look at nodes with no data successors, such as stores.
2827  // These are especially important, due to the heuristics in
2828  // getNodePriority for nodes with no data successors.
2829  if (SU->NumSuccs != 0)
2830  continue;
2831  // For now, only look at nodes with exactly one data predecessor.
2832  if (SU->NumPreds != 1)
2833  continue;
2834  // Avoid prescheduling copies to virtual registers, which don't behave
2835  // like other nodes from the perspective of scheduling heuristics.
2836  if (SDNode *N = SU->getNode())
2837  if (N->getOpcode() == ISD::CopyToReg &&
2839  (cast<RegisterSDNode>(N->getOperand(1))->getReg()))
2840  continue;
2841 
2842  // Locate the single data predecessor.
2843  SUnit *PredSU = nullptr;
2844  for (SUnit::const_pred_iterator II = SU->Preds.begin(),
2845  EE = SU->Preds.end(); II != EE; ++II)
2846  if (!II->isCtrl()) {
2847  PredSU = II->getSUnit();
2848  break;
2849  }
2850  assert(PredSU);
2851 
2852  // Don't rewrite edges that carry physregs, because that requires additional
2853  // support infrastructure.
2854  if (PredSU->hasPhysRegDefs)
2855  continue;
2856  // Short-circuit the case where SU is PredSU's only data successor.
2857  if (PredSU->NumSuccs == 1)
2858  continue;
2859  // Avoid prescheduling to copies from virtual registers, which don't behave
2860  // like other nodes from the perspective of scheduling heuristics.
2861  if (SDNode *N = SU->getNode())
2862  if (N->getOpcode() == ISD::CopyFromReg &&
2864  (cast<RegisterSDNode>(N->getOperand(1))->getReg()))
2865  continue;
2866 
2867  // Perform checks on the successors of PredSU.
2868  for (SUnit::const_succ_iterator II = PredSU->Succs.begin(),
2869  EE = PredSU->Succs.end(); II != EE; ++II) {
2870  SUnit *PredSuccSU = II->getSUnit();
2871  if (PredSuccSU == SU) continue;
2872  // If PredSU has another successor with no data successors, for
2873  // now don't attempt to choose either over the other.
2874  if (PredSuccSU->NumSuccs == 0)
2875  goto outer_loop_continue;
2876  // Don't break physical register dependencies.
2877  if (SU->hasPhysRegClobbers && PredSuccSU->hasPhysRegDefs)
2878  if (canClobberPhysRegDefs(PredSuccSU, SU, TII, TRI))
2879  goto outer_loop_continue;
2880  // Don't introduce graph cycles.
2881  if (scheduleDAG->IsReachable(SU, PredSuccSU))
2882  goto outer_loop_continue;
2883  }
2884 
2885  // Ok, the transformation is safe and the heuristics suggest it is
2886  // profitable. Update the graph.
2887  DEBUG(dbgs() << " Prescheduling SU #" << SU->NodeNum
2888  << " next to PredSU #" << PredSU->NodeNum
2889  << " to guide scheduling in the presence of multiple uses\n");
2890  for (unsigned i = 0; i != PredSU->Succs.size(); ++i) {
2891  SDep Edge = PredSU->Succs[i];
2892  assert(!Edge.isAssignedRegDep());
2893  SUnit *SuccSU = Edge.getSUnit();
2894  if (SuccSU != SU) {
2895  Edge.setSUnit(PredSU);
2896  scheduleDAG->RemovePred(SuccSU, Edge);
2897  scheduleDAG->AddPred(SU, Edge);
2898  Edge.setSUnit(SU);
2899  scheduleDAG->AddPred(SuccSU, Edge);
2900  --i;
2901  }
2902  }
2903  outer_loop_continue:;
2904  }
2905 }
2906 
2907 /// AddPseudoTwoAddrDeps - If two nodes share an operand and one of them uses
2908 /// it as a def&use operand. Add a pseudo control edge from it to the other
2909 /// node (if it won't create a cycle) so the two-address one will be scheduled
2910 /// first (lower in the schedule). If both nodes are two-address, favor the
2911 /// one that has a CopyToReg use (more likely to be a loop induction update).
2912 /// If both are two-address, but one is commutable while the other is not
2913 /// commutable, favor the one that's not commutable.
2914 void RegReductionPQBase::AddPseudoTwoAddrDeps() {
2915  for (unsigned i = 0, e = SUnits->size(); i != e; ++i) {
2916  SUnit *SU = &(*SUnits)[i];
2917  if (!SU->isTwoAddress)
2918  continue;
2919 
2920  SDNode *Node = SU->getNode();
2921  if (!Node || !Node->isMachineOpcode() || SU->getNode()->getGluedNode())
2922  continue;
2923 
2924  bool isLiveOut = hasOnlyLiveOutUses(SU);
2925  unsigned Opc = Node->getMachineOpcode();
2926  const MCInstrDesc &MCID = TII->get(Opc);
2927  unsigned NumRes = MCID.getNumDefs();
2928  unsigned NumOps = MCID.getNumOperands() - NumRes;
2929  for (unsigned j = 0; j != NumOps; ++j) {
2930  if (MCID.getOperandConstraint(j+NumRes, MCOI::TIED_TO) == -1)
2931  continue;
2932  SDNode *DU = SU->getNode()->getOperand(j).getNode();
2933  if (DU->getNodeId() == -1)
2934  continue;
2935  const SUnit *DUSU = &(*SUnits)[DU->getNodeId()];
2936  if (!DUSU) continue;
2937  for (SUnit::const_succ_iterator I = DUSU->Succs.begin(),
2938  E = DUSU->Succs.end(); I != E; ++I) {
2939  if (I->isCtrl()) continue;
2940  SUnit *SuccSU = I->getSUnit();
2941  if (SuccSU == SU)
2942  continue;
2943  // Be conservative. Ignore if nodes aren't at roughly the same
2944  // depth and height.
2945  if (SuccSU->getHeight() < SU->getHeight() &&
2946  (SU->getHeight() - SuccSU->getHeight()) > 1)
2947  continue;
2948  // Skip past COPY_TO_REGCLASS nodes, so that the pseudo edge
2949  // constrains whatever is using the copy, instead of the copy
2950  // itself. In the case that the copy is coalesced, this
2951  // preserves the intent of the pseudo two-address heurietics.
2952  while (SuccSU->Succs.size() == 1 &&
2953  SuccSU->getNode()->isMachineOpcode() &&
2954  SuccSU->getNode()->getMachineOpcode() ==
2956  SuccSU = SuccSU->Succs.front().getSUnit();
2957  // Don't constrain non-instruction nodes.
2958  if (!SuccSU->getNode() || !SuccSU->getNode()->isMachineOpcode())
2959  continue;
2960  // Don't constrain nodes with physical register defs if the
2961  // predecessor can clobber them.
2962  if (SuccSU->hasPhysRegDefs && SU->hasPhysRegClobbers) {
2963  if (canClobberPhysRegDefs(SuccSU, SU, TII, TRI))
2964  continue;
2965  }
2966  // Don't constrain EXTRACT_SUBREG, INSERT_SUBREG, and SUBREG_TO_REG;
2967  // these may be coalesced away. We want them close to their uses.
2968  unsigned SuccOpc = SuccSU->getNode()->getMachineOpcode();
2969  if (SuccOpc == TargetOpcode::EXTRACT_SUBREG ||
2970  SuccOpc == TargetOpcode::INSERT_SUBREG ||
2971  SuccOpc == TargetOpcode::SUBREG_TO_REG)
2972  continue;
2973  if (!canClobberReachingPhysRegUse(SuccSU, SU, scheduleDAG, TII, TRI) &&
2974  (!canClobber(SuccSU, DUSU) ||
2975  (isLiveOut && !hasOnlyLiveOutUses(SuccSU)) ||
2976  (!SU->isCommutable && SuccSU->isCommutable)) &&
2977  !scheduleDAG->IsReachable(SuccSU, SU)) {
2978  DEBUG(dbgs() << " Adding a pseudo-two-addr edge from SU #"
2979  << SU->NodeNum << " to SU #" << SuccSU->NodeNum << "\n");
2980  scheduleDAG->AddPred(SU, SDep(SuccSU, SDep::Artificial));
2981  }
2982  }
2983  }
2984  }
2985 }
2986 
2987 //===----------------------------------------------------------------------===//
2988 // Public Constructor Functions
2989 //===----------------------------------------------------------------------===//
2990 
2993  CodeGenOpt::Level OptLevel) {
2994  const TargetSubtargetInfo &STI = IS->MF->getSubtarget();
2995  const TargetInstrInfo *TII = STI.getInstrInfo();
2996  const TargetRegisterInfo *TRI = STI.getRegisterInfo();
2997 
2998  BURegReductionPriorityQueue *PQ =
2999  new BURegReductionPriorityQueue(*IS->MF, false, false, TII, TRI, nullptr);
3000  ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, PQ, OptLevel);
3001  PQ->setScheduleDAG(SD);
3002  return SD;
3003 }
3004 
3007  CodeGenOpt::Level OptLevel) {
3008  const TargetSubtargetInfo &STI = IS->MF->getSubtarget();
3009  const TargetInstrInfo *TII = STI.getInstrInfo();
3010  const TargetRegisterInfo *TRI = STI.getRegisterInfo();
3011 
3012  SrcRegReductionPriorityQueue *PQ =
3013  new SrcRegReductionPriorityQueue(*IS->MF, false, true, TII, TRI, nullptr);
3014  ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, false, PQ, OptLevel);
3015  PQ->setScheduleDAG(SD);
3016  return SD;
3017 }
3018 
3021  CodeGenOpt::Level OptLevel) {
3022  const TargetSubtargetInfo &STI = IS->MF->getSubtarget();
3023  const TargetInstrInfo *TII = STI.getInstrInfo();
3024  const TargetRegisterInfo *TRI = STI.getRegisterInfo();
3025  const TargetLowering *TLI = IS->TLI;
3026 
3027  HybridBURRPriorityQueue *PQ =
3028  new HybridBURRPriorityQueue(*IS->MF, true, false, TII, TRI, TLI);
3029 
3030  ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, PQ, OptLevel);
3031  PQ->setScheduleDAG(SD);
3032  return SD;
3033 }
3034 
3037  CodeGenOpt::Level OptLevel) {
3038  const TargetSubtargetInfo &STI = IS->MF->getSubtarget();
3039  const TargetInstrInfo *TII = STI.getInstrInfo();
3040  const TargetRegisterInfo *TRI = STI.getRegisterInfo();
3041  const TargetLowering *TLI = IS->TLI;
3042 
3043  ILPBURRPriorityQueue *PQ =
3044  new ILPBURRPriorityQueue(*IS->MF, true, false, TII, TRI, TLI);
3045  ScheduleDAGRRList *SD = new ScheduleDAGRRList(*IS->MF, true, PQ, OptLevel);
3046  PQ->setScheduleDAG(SD);
3047  return SD;
3048 }
bool isCtrl() const
isCtrl - Shorthand for getKind() != SDep::Data.
Definition: ScheduleDAG.h:175
const uint16_t * getImplicitDefs() const
Return a list of registers that are potentially written by any instance of this machine instruction...
Definition: MCInstrDesc.h:497
unsigned NumPreds
Definition: ScheduleDAG.h:285
bool isSucc(SUnit *N)
isSucc - Test if node N is a successor of this node.
Definition: ScheduleDAG.h:466
STATISTIC(NumFunctions,"Total number of functions")
static int BUCompareLatency(SUnit *left, SUnit *right, bool checkPref, RegReductionPQBase *SPQ)
bool isCommutable() const
Return true if this may be a 2- or 3-address instruction (of the form "X = op Y, Z, ..."), which produces the same result if Y and Z are exchanged.
Definition: MCInstrDesc.h:388
static bool canEnableCoalescing(SUnit *SU)
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:191
static cl::opt< bool > DisableSchedRegPressure("disable-sched-reg-pressure", cl::Hidden, cl::init(false), cl::desc("Disable regpressure priority in sched=list-ilp"))
static cl::opt< unsigned > AvgIPC("sched-avg-ipc", cl::Hidden, cl::init(1), cl::desc("Average inst/cycle whan no target itinerary exists."))
bool isTwoAddress
Definition: ScheduleDAG.h:296
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:138
static void GetCostForDef(const ScheduleDAGSDNodes::RegDefIter &RegDefPos, const TargetLowering *TLI, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, unsigned &RegClass, unsigned &Cost, const MachineFunction &MF)
GetCostForDef - Looks up the register class and cost for a given definition.
static bool isVirtualRegister(unsigned Reg)
isVirtualRegister - Return true if the specified register number is in the virtual register namespace...
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getID() const
getID() - Return the register class ID number.
static bool isClobberKind(unsigned Flag)
Definition: InlineAsm.h:321
void setSUnit(SUnit *SU)
Definition: ScheduleDAG.h:165
static void CheckForLiveRegDef(SUnit *SU, unsigned Reg, std::vector< SUnit * > &LiveRegDefs, SmallSet< unsigned, 4 > &RegAdded, SmallVectorImpl< unsigned > &LRegs, const TargetRegisterInfo *TRI)
CheckForLiveRegDef - Return true and update live register vector if the specified register def of the...
static bool canClobberReachingPhysRegUse(const SUnit *DepSU, const SUnit *SU, ScheduleDAGRRList *scheduleDAG, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
canClobberReachingPhysRegUse - True if SU would clobber one of it's successor's explicit physregs who...
const SDValue & getOperand(unsigned Num) const
void removePred(const SDep &D)
removePred - This removes the specified edge as a pred of the current node if it exists.
static cl::opt< bool > DisableSchedVRegCycle("disable-sched-vrcycle", cl::Hidden, cl::init(false), cl::desc("Disable virtual register cycle interference checks"))
void setNodeId(int Id)
Set unique node id.
static unsigned CalcNodeSethiUllmanNumber(const SUnit *SU, std::vector< unsigned > &SUNumbers)
CalcNodeSethiUllmanNumber - Compute Sethi Ullman number.
bool isAssignedRegDep() const
isAssignedRegDep - Test if this is a Data dependence that is associated with a register.
Definition: ScheduleDAG.h:228
static bool hasVRegCycleUse(const SUnit *SU)
const TargetRegisterClass * CopyDstRC
Definition: ScheduleDAG.h:320
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:45
virtual const TargetRegisterClass * getRepRegClassFor(MVT VT) const
Return the 'representative' register class for the specified value type.
MachineFunction * MF
SmallVector< SDep, 4 > Preds
Definition: ScheduleDAG.h:275
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, bool gen_crash_diag=true)
Reports a serious error, calling any installed error handler.
static SDNode * FindCallSeqStart(SDNode *N, unsigned &NestLevel, unsigned &MaxNest, const TargetInstrInfo *TII)
FindCallSeqStart - Starting from the (lowered) CALLSEQ_END node, locate the corresponding (lowered) C...
bool isScheduled
Definition: ScheduleDAG.h:303
SchedulingPriorityQueue - This interface is used to plug different priorities computation algorithms ...
Definition: ScheduleDAG.h:501
unsigned getHeight() const
getHeight - Return the height of this node, which is the length of the maximum path down to any node ...
Definition: ScheduleDAG.h:431
unsigned getCallFrameDestroyOpcode() const
static int checkSpecialNodes(const SUnit *left, const SUnit *right)
unsigned NumSuccs
Definition: ScheduleDAG.h:286
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
unsigned NumSuccsLeft
Definition: ScheduleDAG.h:288
const HexagonInstrInfo * TII
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
const TargetRegisterClass * getRegClass(unsigned i) const
getRegClass - Returns the register class associated with the enumeration value.
const TargetLowering * TLI
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
Regular data dependence (aka true-dependence).
Definition: ScheduleDAG.h:49
CopyToReg - This node has three operands: a chain, a register number to set to this value...
Definition: ISDOpcodes.h:161
const TargetRegisterClass * getRegClass(unsigned Reg) const
getRegClass - Return the register class of the specified virtual register.
Reg
All possible values of the reg field in the ModR/M byte.
bool hasPhysRegDefs
Definition: ScheduleDAG.h:299
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
static bool isRegDefEarlyClobberKind(unsigned Flag)
Definition: InlineAsm.h:318
static const uint32_t * getNodeRegMask(const SDNode *N)
getNodeRegMask - Returns the register mask attached to an SDNode, if any.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:571
SUnit * OrigNode
Definition: ScheduleDAG.h:268
unsigned getIROrder() const
Return the node ordering.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
ScheduleDAGSDNodes * createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createHybridListDAGScheduler - This creates a bottom up register pressure aware list scheduler that m...
unsigned getCallFrameSetupOpcode() const
These methods return the opcode of the frame setup/destroy instructions if they exist (-1 otherwise)...
bool isPending
Definition: ScheduleDAG.h:301
static bool isRegDefKind(unsigned Flag)
Definition: InlineAsm.h:315
ScheduleDAGSDNodes * createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up list scheduler that schedules nodes in source c...
static RegisterScheduler burrListDAGScheduler("list-burr","Bottom-up register reduction list scheduling", createBURRListDAGScheduler)
TargetInstrInfo - Interface to description of machine instruction set.
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:711
SDep - Scheduling dependency.
Definition: ScheduleDAG.h:45
SDNode * getNode() const
get the SDNode which holds the desired result
IMPLICIT_DEF - This is the MachineInstr-level equivalent of undef.
Definition: TargetOpcodes.h:52
void setHeightToAtLeast(unsigned NewHeight)
setDepthToAtLeast - If NewDepth is greater than this node's depth value, set it to be the new height ...
bool isAvailable()
Definition: Compression.cpp:48
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
static RegisterScheduler hybridListDAGScheduler("list-hybrid","Bottom-up register pressure aware list scheduling ""which tries to balance latency and register pressure", createHybridListDAGScheduler)
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
bool regsOverlap(unsigned regA, unsigned regB) const
regsOverlap - Returns true if the two registers are equal or alias each other.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
static cl::opt< bool > DisableSchedStalls("disable-sched-stalls", cl::Hidden, cl::init(true), cl::desc("Disable no-stall priority in sched=list-ilp"))
MVT - Machine Value Type.
HazardRecognizer - This determines whether or not an instruction can be issued this cycle...
SDNode * getNode() const
getNode - Return the representative SDNode for this SUnit.
Definition: ScheduleDAG.h:388
unsigned short Latency
Definition: ScheduleDAG.h:292
ScheduleDAGSDNodes * createBURRListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createBURRListDAGScheduler - This creates a bottom up register usage reduction list scheduler...
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:32
static bool isOperandOf(const SUnit *SU, SDNode *N)
static RegisterScheduler ILPListDAGScheduler("list-ilp","Bottom-up register pressure aware list scheduling ""which tries to balance ILP and register pressure", createILPListDAGScheduler)
static unsigned calcMaxScratches(const SUnit *SU)
calcMaxScratches - Returns an cost estimate of the worse case requirement for scratch registers...
static bool IsChainDependent(SDNode *Outer, SDNode *Inner, unsigned NestLevel, const TargetInstrInfo *TII)
IsChainDependent - Test if Outer is reachable from Inner through chain dependencies.
static bool BUHasStall(SUnit *SU, int Height, RegReductionPQBase *SPQ)
INSERT_SUBREG - This instruction takes three operands: a register that has subregisters, a register providing an insert value, and a subregister index.
Definition: TargetOpcodes.h:49
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
Definition: InlineAsm.h:332
static cl::opt< bool > DisableSchedCriticalPath("disable-sched-critical-path", cl::Hidden, cl::init(false), cl::desc("Disable critical path priority in sched=list-ilp"))
bool isVRegCycle
Definition: ScheduleDAG.h:293
MCRegAliasIterator enumerates all registers aliasing Reg.
unsigned getLatency() const
getLatency - Return the latency value for this edge, which roughly means the minimum number of cycles...
Definition: ScheduleDAG.h:150
for(unsigned i=0, e=MI->getNumOperands();i!=e;++i)
static bool canClobberPhysRegDefs(const SUnit *SuccSU, const SUnit *SU, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
canClobberPhysRegDefs - True if SU would clobber one of SuccSU's physical register defs...
static void resetVRegCycle(SUnit *SU)
static cl::opt< bool > DisableSchedLiveUses("disable-sched-live-uses", cl::Hidden, cl::init(true), cl::desc("Disable live use priority in sched=list-ilp"))
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:69
static bool hasOnlyLiveInOpers(const SUnit *SU)
hasOnlyLiveInOpers - Return true if SU has only value predecessors that are CopyFromReg from a virtua...
static unsigned closestSucc(const SUnit *SU)
closestSucc - Returns the scheduled cycle of the successor which is closest to the current cycle...
Sched::Preference SchedulingPref
Definition: ScheduleDAG.h:309
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
static void CheckForLiveRegDefMasked(SUnit *SU, const uint32_t *RegMask, std::vector< SUnit * > &LiveRegDefs, SmallSet< unsigned, 4 > &RegAdded, SmallVectorImpl< unsigned > &LRegs)
CheckForLiveRegDefMasked - Check for any live physregs that are clobbered by RegMask, and add them to LRegs.
RegDefIter - In place iteration over the values defined by an SUnit.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specific constraint if it is set.
Definition: MCInstrDesc.h:162
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:576
REG_SEQUENCE - This variadic instruction is used to form a register that represents a consecutive seq...
Definition: TargetOpcodes.h:82
static bool BURRSort(SUnit *left, SUnit *right, RegReductionPQBase *SPQ)
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
bool isCommutable
Definition: ScheduleDAG.h:297
EXTRACT_SUBREG - This instruction takes two operands: a register that has subregisters, and a subregister index.
Definition: TargetOpcodes.h:41
ScheduleDAGSDNodes * createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level)
createILPListDAGScheduler - This creates a bottom up register pressure aware list scheduler that trie...
static cl::opt< bool > DisableSchedHeight("disable-sched-height", cl::Hidden, cl::init(false), cl::desc("Disable scheduled-height priority in sched=list-ilp"))
static void initVRegCycle(SUnit *SU)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
SI Lower i1 Copies
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
const TargetRegisterClass * CopySrcRC
Definition: ScheduleDAG.h:321
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:576
static bool clobbersPhysReg(const uint32_t *RegMask, unsigned PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
SUnit * getSUnit() const
Definition: ScheduleDAG.h:160
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
unsigned getDepth() const
getDepth - Return the depth of this node, which is the length of the maximum path up to any node whic...
Definition: ScheduleDAG.h:423
iterator_range< value_op_iterator > op_values() const
void setLatency(unsigned Lat)
setLatency - Set the latency for this edge.
Definition: ScheduleDAG.h:155
TargetSubtargetInfo - Generic base class for all target subtargets.
bool isAvailable
Definition: ScheduleDAG.h:302
unsigned NodeQueueId
Definition: ScheduleDAG.h:284
static bool isPhysicalRegister(unsigned Reg)
isPhysicalRegister - Return true if the specified register number is in the physical register namespa...
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
static bool hasOnlyLiveOutUses(const SUnit *SU)
hasOnlyLiveOutUses - Return true if SU has only value successors that are CopyToReg to a virtual regi...
static cl::opt< bool > DisableSchedPhysRegJoin("disable-sched-physreg-join", cl::Hidden, cl::init(false), cl::desc("Disable physreg def-use affinity"))
COPY_TO_REGCLASS - This instruction is a placeholder for a plain register-to-register copy into a spe...
Definition: TargetOpcodes.h:66
static cl::opt< int > MaxReorderWindow("max-sched-reorder", cl::Hidden, cl::init(6), cl::desc("Number of instructions to allow ahead of the critical path ""in sched=list-ilp"))
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:651
Sequence
A sequence of states that a pointer may go through in which an objc_retain and objc_release are actua...
Definition: PtrState.h:37
unsigned short NumRegDefsLeft
Definition: ScheduleDAG.h:291
static cl::opt< bool > Disable2AddrHack("disable-2addr-hack", cl::Hidden, cl::init(true), cl::desc("Disable scheduler's two-address hack"))
static MVT getPhysicalRegisterVT(SDNode *N, unsigned Reg, const TargetInstrInfo *TII)
getPhysicalRegisterVT - Returns the ValueType of the physical register definition of the specified no...
static cl::opt< bool > DisableSchedCycles("disable-sched-cycles", cl::Hidden, cl::init(false), cl::desc("Disable cycle-level precision during preRA scheduling"))
int getNodeId() const
Return the unique node id.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:166
static RegisterScheduler sourceListDAGScheduler("source","Similar to list-burr but schedules in source ""order when possible", createSourceListDAGScheduler)
virtual unsigned getRegPressureLimit(const TargetRegisterClass *RC, MachineFunction &MF) const
getRegPressureLimit - Return the register pressure "high water mark" for the specific register class...
unsigned NodeNum
Definition: ScheduleDAG.h:283
virtual ScheduleHazardRecognizer * CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, const ScheduleDAG *DAG) const
Allocate and return a hazard recognizer to use for this target when scheduling the machine instructio...
void setHeightDirty()
setHeightDirty - Set a flag in this node to indicate that its stored Height value will require recomp...
const uint16_t * ImplicitDefs
Definition: MCInstrDesc.h:148
bool addPred(const SDep &D, bool Required=true)
addPred - This adds the specified edge as a pred of the current node if not already.
Definition: ScheduleDAG.cpp:65
virtual const TargetInstrInfo * getInstrInfo() const
SmallVector< SDep, 4 > Succs
Definition: ScheduleDAG.h:276
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:185
Arbitrary strong DAG edge (no real dependence).
Definition: ScheduleDAG.h:68
#define DEBUG(X)
Definition: Debug.h:92
virtual uint8_t getRepRegClassCostFor(MVT VT) const
Return the cost of the 'representative' register class for the specified value type.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
const TargetRegisterClass * getRegClass(const MCInstrDesc &TID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const
Given a machine instruction descriptor, returns the register class constraint for OpNum...
bool isScheduleLow
Definition: ScheduleDAG.h:305
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:188
SUBREG_TO_REG - This instruction is similar to INSERT_SUBREG except that the first operand is an imme...
Definition: TargetOpcodes.h:58
ScheduleDAGTopologicalSort is a class that computes a topological ordering for SUnits and provides me...
Definition: ScheduleDAG.h:691
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
bool hasPhysRegClobbers
Definition: ScheduleDAG.h:300
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
const TargetRegisterClass *const * regclass_iterator
void dump(const ScheduleDAG *G) const
SUnit - Scheduling unit.
SUnit - Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:261
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
This file describes how to lower LLVM code to machine code.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.