LLVM  3.7.0
ScheduleDAGFast.cpp
Go to the documentation of this file.
1 //===----- ScheduleDAGFast.cpp - Fast poor list scheduler -----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements a fast scheduler.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "InstrEmitter.h"
16 #include "ScheduleDAGSDNodes.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/Statistic.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/InlineAsm.h"
23 #include "llvm/Support/Debug.h"
28 using namespace llvm;
29 
30 #define DEBUG_TYPE "pre-RA-sched"
31 
32 STATISTIC(NumUnfolds, "Number of nodes unfolded");
33 STATISTIC(NumDups, "Number of duplicated nodes");
34 STATISTIC(NumPRCopies, "Number of physical copies");
35 
36 static RegisterScheduler
37  fastDAGScheduler("fast", "Fast suboptimal list scheduling",
39 static RegisterScheduler
40  linearizeDAGScheduler("linearize", "Linearize DAG, no scheduling",
42 
43 
44 namespace {
45  /// FastPriorityQueue - A degenerate priority queue that considers
46  /// all nodes to have the same priority.
47  ///
48  struct FastPriorityQueue {
50 
51  bool empty() const { return Queue.empty(); }
52 
53  void push(SUnit *U) {
54  Queue.push_back(U);
55  }
56 
57  SUnit *pop() {
58  if (empty()) return nullptr;
59  SUnit *V = Queue.back();
60  Queue.pop_back();
61  return V;
62  }
63  };
64 
65 //===----------------------------------------------------------------------===//
66 /// ScheduleDAGFast - The actual "fast" list scheduler implementation.
67 ///
68 class ScheduleDAGFast : public ScheduleDAGSDNodes {
69 private:
70  /// AvailableQueue - The priority queue to use for the available SUnits.
71  FastPriorityQueue AvailableQueue;
72 
73  /// LiveRegDefs - A set of physical registers and their definition
74  /// that are "live". These nodes must be scheduled before any other nodes that
75  /// modifies the registers can be scheduled.
76  unsigned NumLiveRegs;
77  std::vector<SUnit*> LiveRegDefs;
78  std::vector<unsigned> LiveRegCycles;
79 
80 public:
81  ScheduleDAGFast(MachineFunction &mf)
82  : ScheduleDAGSDNodes(mf) {}
83 
84  void Schedule() override;
85 
86  /// AddPred - adds a predecessor edge to SUnit SU.
87  /// This returns true if this is a new predecessor.
88  void AddPred(SUnit *SU, const SDep &D) {
89  SU->addPred(D);
90  }
91 
92  /// RemovePred - removes a predecessor edge from SUnit SU.
93  /// This returns true if an edge was removed.
94  void RemovePred(SUnit *SU, const SDep &D) {
95  SU->removePred(D);
96  }
97 
98 private:
99  void ReleasePred(SUnit *SU, SDep *PredEdge);
100  void ReleasePredecessors(SUnit *SU, unsigned CurCycle);
101  void ScheduleNodeBottomUp(SUnit*, unsigned);
102  SUnit *CopyAndMoveSuccessors(SUnit*);
103  void InsertCopiesAndMoveSuccs(SUnit*, unsigned,
104  const TargetRegisterClass*,
105  const TargetRegisterClass*,
107  bool DelayForLiveRegsBottomUp(SUnit*, SmallVectorImpl<unsigned>&);
108  void ListScheduleBottomUp();
109 
110  /// forceUnitLatencies - The fast scheduler doesn't care about real latencies.
111  bool forceUnitLatencies() const override { return true; }
112 };
113 } // end anonymous namespace
114 
115 
116 /// Schedule - Schedule the DAG using list scheduling.
117 void ScheduleDAGFast::Schedule() {
118  DEBUG(dbgs() << "********** List Scheduling **********\n");
119 
120  NumLiveRegs = 0;
121  LiveRegDefs.resize(TRI->getNumRegs(), nullptr);
122  LiveRegCycles.resize(TRI->getNumRegs(), 0);
123 
124  // Build the scheduling graph.
125  BuildSchedGraph(nullptr);
126 
127  DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
128  SUnits[su].dumpAll(this));
129 
130  // Execute the actual scheduling loop.
131  ListScheduleBottomUp();
132 }
133 
134 //===----------------------------------------------------------------------===//
135 // Bottom-Up Scheduling
136 //===----------------------------------------------------------------------===//
137 
138 /// ReleasePred - Decrement the NumSuccsLeft count of a predecessor. Add it to
139 /// the AvailableQueue if the count reaches zero. Also update its cycle bound.
140 void ScheduleDAGFast::ReleasePred(SUnit *SU, SDep *PredEdge) {
141  SUnit *PredSU = PredEdge->getSUnit();
142 
143 #ifndef NDEBUG
144  if (PredSU->NumSuccsLeft == 0) {
145  dbgs() << "*** Scheduling failed! ***\n";
146  PredSU->dump(this);
147  dbgs() << " has been released too many times!\n";
148  llvm_unreachable(nullptr);
149  }
150 #endif
151  --PredSU->NumSuccsLeft;
152 
153  // If all the node's successors are scheduled, this node is ready
154  // to be scheduled. Ignore the special EntrySU node.
155  if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) {
156  PredSU->isAvailable = true;
157  AvailableQueue.push(PredSU);
158  }
159 }
160 
161 void ScheduleDAGFast::ReleasePredecessors(SUnit *SU, unsigned CurCycle) {
162  // Bottom up: release predecessors
163  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
164  I != E; ++I) {
165  ReleasePred(SU, &*I);
166  if (I->isAssignedRegDep()) {
167  // This is a physical register dependency and it's impossible or
168  // expensive to copy the register. Make sure nothing that can
169  // clobber the register is scheduled between the predecessor and
170  // this node.
171  if (!LiveRegDefs[I->getReg()]) {
172  ++NumLiveRegs;
173  LiveRegDefs[I->getReg()] = I->getSUnit();
174  LiveRegCycles[I->getReg()] = CurCycle;
175  }
176  }
177  }
178 }
179 
180 /// ScheduleNodeBottomUp - Add the node to the schedule. Decrement the pending
181 /// count of its predecessors. If a predecessor pending count is zero, add it to
182 /// the Available queue.
183 void ScheduleDAGFast::ScheduleNodeBottomUp(SUnit *SU, unsigned CurCycle) {
184  DEBUG(dbgs() << "*** Scheduling [" << CurCycle << "]: ");
185  DEBUG(SU->dump(this));
186 
187  assert(CurCycle >= SU->getHeight() && "Node scheduled below its height!");
188  SU->setHeightToAtLeast(CurCycle);
189  Sequence.push_back(SU);
190 
191  ReleasePredecessors(SU, CurCycle);
192 
193  // Release all the implicit physical register defs that are live.
194  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
195  I != E; ++I) {
196  if (I->isAssignedRegDep()) {
197  if (LiveRegCycles[I->getReg()] == I->getSUnit()->getHeight()) {
198  assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!");
199  assert(LiveRegDefs[I->getReg()] == SU &&
200  "Physical register dependency violated?");
201  --NumLiveRegs;
202  LiveRegDefs[I->getReg()] = nullptr;
203  LiveRegCycles[I->getReg()] = 0;
204  }
205  }
206  }
207 
208  SU->isScheduled = true;
209 }
210 
211 /// CopyAndMoveSuccessors - Clone the specified node and move its scheduled
212 /// successors to the newly created node.
213 SUnit *ScheduleDAGFast::CopyAndMoveSuccessors(SUnit *SU) {
214  if (SU->getNode()->getGluedNode())
215  return nullptr;
216 
217  SDNode *N = SU->getNode();
218  if (!N)
219  return nullptr;
220 
221  SUnit *NewSU;
222  bool TryUnfold = false;
223  for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
224  MVT VT = N->getSimpleValueType(i);
225  if (VT == MVT::Glue)
226  return nullptr;
227  else if (VT == MVT::Other)
228  TryUnfold = true;
229  }
230  for (const SDValue &Op : N->op_values()) {
231  MVT VT = Op.getNode()->getSimpleValueType(Op.getResNo());
232  if (VT == MVT::Glue)
233  return nullptr;
234  }
235 
236  if (TryUnfold) {
237  SmallVector<SDNode*, 2> NewNodes;
238  if (!TII->unfoldMemoryOperand(*DAG, N, NewNodes))
239  return nullptr;
240 
241  DEBUG(dbgs() << "Unfolding SU # " << SU->NodeNum << "\n");
242  assert(NewNodes.size() == 2 && "Expected a load folding node!");
243 
244  N = NewNodes[1];
245  SDNode *LoadNode = NewNodes[0];
246  unsigned NumVals = N->getNumValues();
247  unsigned OldNumVals = SU->getNode()->getNumValues();
248  for (unsigned i = 0; i != NumVals; ++i)
249  DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i));
250  DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals-1),
251  SDValue(LoadNode, 1));
252 
253  SUnit *NewSU = newSUnit(N);
254  assert(N->getNodeId() == -1 && "Node already inserted!");
255  N->setNodeId(NewSU->NodeNum);
256 
257  const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
258  for (unsigned i = 0; i != MCID.getNumOperands(); ++i) {
259  if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) {
260  NewSU->isTwoAddress = true;
261  break;
262  }
263  }
264  if (MCID.isCommutable())
265  NewSU->isCommutable = true;
266 
267  // LoadNode may already exist. This can happen when there is another
268  // load from the same location and producing the same type of value
269  // but it has different alignment or volatileness.
270  bool isNewLoad = true;
271  SUnit *LoadSU;
272  if (LoadNode->getNodeId() != -1) {
273  LoadSU = &SUnits[LoadNode->getNodeId()];
274  isNewLoad = false;
275  } else {
276  LoadSU = newSUnit(LoadNode);
277  LoadNode->setNodeId(LoadSU->NodeNum);
278  }
279 
280  SDep ChainPred;
281  SmallVector<SDep, 4> ChainSuccs;
282  SmallVector<SDep, 4> LoadPreds;
283  SmallVector<SDep, 4> NodePreds;
284  SmallVector<SDep, 4> NodeSuccs;
285  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
286  I != E; ++I) {
287  if (I->isCtrl())
288  ChainPred = *I;
289  else if (I->getSUnit()->getNode() &&
290  I->getSUnit()->getNode()->isOperandOf(LoadNode))
291  LoadPreds.push_back(*I);
292  else
293  NodePreds.push_back(*I);
294  }
295  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
296  I != E; ++I) {
297  if (I->isCtrl())
298  ChainSuccs.push_back(*I);
299  else
300  NodeSuccs.push_back(*I);
301  }
302 
303  if (ChainPred.getSUnit()) {
304  RemovePred(SU, ChainPred);
305  if (isNewLoad)
306  AddPred(LoadSU, ChainPred);
307  }
308  for (unsigned i = 0, e = LoadPreds.size(); i != e; ++i) {
309  const SDep &Pred = LoadPreds[i];
310  RemovePred(SU, Pred);
311  if (isNewLoad) {
312  AddPred(LoadSU, Pred);
313  }
314  }
315  for (unsigned i = 0, e = NodePreds.size(); i != e; ++i) {
316  const SDep &Pred = NodePreds[i];
317  RemovePred(SU, Pred);
318  AddPred(NewSU, Pred);
319  }
320  for (unsigned i = 0, e = NodeSuccs.size(); i != e; ++i) {
321  SDep D = NodeSuccs[i];
322  SUnit *SuccDep = D.getSUnit();
323  D.setSUnit(SU);
324  RemovePred(SuccDep, D);
325  D.setSUnit(NewSU);
326  AddPred(SuccDep, D);
327  }
328  for (unsigned i = 0, e = ChainSuccs.size(); i != e; ++i) {
329  SDep D = ChainSuccs[i];
330  SUnit *SuccDep = D.getSUnit();
331  D.setSUnit(SU);
332  RemovePred(SuccDep, D);
333  if (isNewLoad) {
334  D.setSUnit(LoadSU);
335  AddPred(SuccDep, D);
336  }
337  }
338  if (isNewLoad) {
339  SDep D(LoadSU, SDep::Barrier);
340  D.setLatency(LoadSU->Latency);
341  AddPred(NewSU, D);
342  }
343 
344  ++NumUnfolds;
345 
346  if (NewSU->NumSuccsLeft == 0) {
347  NewSU->isAvailable = true;
348  return NewSU;
349  }
350  SU = NewSU;
351  }
352 
353  DEBUG(dbgs() << "Duplicating SU # " << SU->NodeNum << "\n");
354  NewSU = Clone(SU);
355 
356  // New SUnit has the exact same predecessors.
357  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
358  I != E; ++I)
359  if (!I->isArtificial())
360  AddPred(NewSU, *I);
361 
362  // Only copy scheduled successors. Cut them from old node's successor
363  // list and move them over.
365  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
366  I != E; ++I) {
367  if (I->isArtificial())
368  continue;
369  SUnit *SuccSU = I->getSUnit();
370  if (SuccSU->isScheduled) {
371  SDep D = *I;
372  D.setSUnit(NewSU);
373  AddPred(SuccSU, D);
374  D.setSUnit(SU);
375  DelDeps.push_back(std::make_pair(SuccSU, D));
376  }
377  }
378  for (unsigned i = 0, e = DelDeps.size(); i != e; ++i)
379  RemovePred(DelDeps[i].first, DelDeps[i].second);
380 
381  ++NumDups;
382  return NewSU;
383 }
384 
385 /// InsertCopiesAndMoveSuccs - Insert register copies and move all
386 /// scheduled successors of the given SUnit to the last copy.
387 void ScheduleDAGFast::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg,
388  const TargetRegisterClass *DestRC,
389  const TargetRegisterClass *SrcRC,
391  SUnit *CopyFromSU = newSUnit(static_cast<SDNode *>(nullptr));
392  CopyFromSU->CopySrcRC = SrcRC;
393  CopyFromSU->CopyDstRC = DestRC;
394 
395  SUnit *CopyToSU = newSUnit(static_cast<SDNode *>(nullptr));
396  CopyToSU->CopySrcRC = DestRC;
397  CopyToSU->CopyDstRC = SrcRC;
398 
399  // Only copy scheduled successors. Cut them from old node's successor
400  // list and move them over.
402  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
403  I != E; ++I) {
404  if (I->isArtificial())
405  continue;
406  SUnit *SuccSU = I->getSUnit();
407  if (SuccSU->isScheduled) {
408  SDep D = *I;
409  D.setSUnit(CopyToSU);
410  AddPred(SuccSU, D);
411  DelDeps.push_back(std::make_pair(SuccSU, *I));
412  }
413  }
414  for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) {
415  RemovePred(DelDeps[i].first, DelDeps[i].second);
416  }
417  SDep FromDep(SU, SDep::Data, Reg);
418  FromDep.setLatency(SU->Latency);
419  AddPred(CopyFromSU, FromDep);
420  SDep ToDep(CopyFromSU, SDep::Data, 0);
421  ToDep.setLatency(CopyFromSU->Latency);
422  AddPred(CopyToSU, ToDep);
423 
424  Copies.push_back(CopyFromSU);
425  Copies.push_back(CopyToSU);
426 
427  ++NumPRCopies;
428 }
429 
430 /// getPhysicalRegisterVT - Returns the ValueType of the physical register
431 /// definition of the specified node.
432 /// FIXME: Move to SelectionDAG?
433 static MVT getPhysicalRegisterVT(SDNode *N, unsigned Reg,
434  const TargetInstrInfo *TII) {
435  unsigned NumRes;
436  if (N->getOpcode() == ISD::CopyFromReg) {
437  // CopyFromReg has: "chain, Val, glue" so operand 1 gives the type.
438  NumRes = 1;
439  } else {
440  const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
441  assert(MCID.ImplicitDefs && "Physical reg def must be in implicit def list!");
442  NumRes = MCID.getNumDefs();
443  for (const uint16_t *ImpDef = MCID.getImplicitDefs(); *ImpDef; ++ImpDef) {
444  if (Reg == *ImpDef)
445  break;
446  ++NumRes;
447  }
448  }
449  return N->getSimpleValueType(NumRes);
450 }
451 
452 /// CheckForLiveRegDef - Return true and update live register vector if the
453 /// specified register def of the specified SUnit clobbers any "live" registers.
454 static bool CheckForLiveRegDef(SUnit *SU, unsigned Reg,
455  std::vector<SUnit*> &LiveRegDefs,
456  SmallSet<unsigned, 4> &RegAdded,
458  const TargetRegisterInfo *TRI) {
459  bool Added = false;
460  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) {
461  if (LiveRegDefs[*AI] && LiveRegDefs[*AI] != SU) {
462  if (RegAdded.insert(*AI).second) {
463  LRegs.push_back(*AI);
464  Added = true;
465  }
466  }
467  }
468  return Added;
469 }
470 
471 /// DelayForLiveRegsBottomUp - Returns true if it is necessary to delay
472 /// scheduling of the given node to satisfy live physical register dependencies.
473 /// If the specific node is the last one that's available to schedule, do
474 /// whatever is necessary (i.e. backtracking or cloning) to make it possible.
475 bool ScheduleDAGFast::DelayForLiveRegsBottomUp(SUnit *SU,
477  if (NumLiveRegs == 0)
478  return false;
479 
480  SmallSet<unsigned, 4> RegAdded;
481  // If this node would clobber any "live" register, then it's not ready.
482  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
483  I != E; ++I) {
484  if (I->isAssignedRegDep()) {
485  CheckForLiveRegDef(I->getSUnit(), I->getReg(), LiveRegDefs,
486  RegAdded, LRegs, TRI);
487  }
488  }
489 
490  for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) {
491  if (Node->getOpcode() == ISD::INLINEASM) {
492  // Inline asm can clobber physical defs.
493  unsigned NumOps = Node->getNumOperands();
494  if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
495  --NumOps; // Ignore the glue operand.
496 
497  for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
498  unsigned Flags =
499  cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
500  unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
501 
502  ++i; // Skip the ID value.
503  if (InlineAsm::isRegDefKind(Flags) ||
505  InlineAsm::isClobberKind(Flags)) {
506  // Check for def of register or earlyclobber register.
507  for (; NumVals; --NumVals, ++i) {
508  unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
510  CheckForLiveRegDef(SU, Reg, LiveRegDefs, RegAdded, LRegs, TRI);
511  }
512  } else
513  i += NumVals;
514  }
515  continue;
516  }
517  if (!Node->isMachineOpcode())
518  continue;
519  const MCInstrDesc &MCID = TII->get(Node->getMachineOpcode());
520  if (!MCID.ImplicitDefs)
521  continue;
522  for (const uint16_t *Reg = MCID.getImplicitDefs(); *Reg; ++Reg) {
523  CheckForLiveRegDef(SU, *Reg, LiveRegDefs, RegAdded, LRegs, TRI);
524  }
525  }
526  return !LRegs.empty();
527 }
528 
529 
530 /// ListScheduleBottomUp - The main loop of list scheduling for bottom-up
531 /// schedulers.
532 void ScheduleDAGFast::ListScheduleBottomUp() {
533  unsigned CurCycle = 0;
534 
535  // Release any predecessors of the special Exit node.
536  ReleasePredecessors(&ExitSU, CurCycle);
537 
538  // Add root to Available queue.
539  if (!SUnits.empty()) {
540  SUnit *RootSU = &SUnits[DAG->getRoot().getNode()->getNodeId()];
541  assert(RootSU->Succs.empty() && "Graph root shouldn't have successors!");
542  RootSU->isAvailable = true;
543  AvailableQueue.push(RootSU);
544  }
545 
546  // While Available queue is not empty, grab the node with the highest
547  // priority. If it is not ready put it back. Schedule the node.
548  SmallVector<SUnit*, 4> NotReady;
550  Sequence.reserve(SUnits.size());
551  while (!AvailableQueue.empty()) {
552  bool Delayed = false;
553  LRegsMap.clear();
554  SUnit *CurSU = AvailableQueue.pop();
555  while (CurSU) {
557  if (!DelayForLiveRegsBottomUp(CurSU, LRegs))
558  break;
559  Delayed = true;
560  LRegsMap.insert(std::make_pair(CurSU, LRegs));
561 
562  CurSU->isPending = true; // This SU is not in AvailableQueue right now.
563  NotReady.push_back(CurSU);
564  CurSU = AvailableQueue.pop();
565  }
566 
567  // All candidates are delayed due to live physical reg dependencies.
568  // Try code duplication or inserting cross class copies
569  // to resolve it.
570  if (Delayed && !CurSU) {
571  if (!CurSU) {
572  // Try duplicating the nodes that produces these
573  // "expensive to copy" values to break the dependency. In case even
574  // that doesn't work, insert cross class copies.
575  SUnit *TrySU = NotReady[0];
576  SmallVectorImpl<unsigned> &LRegs = LRegsMap[TrySU];
577  assert(LRegs.size() == 1 && "Can't handle this yet!");
578  unsigned Reg = LRegs[0];
579  SUnit *LRDef = LiveRegDefs[Reg];
580  MVT VT = getPhysicalRegisterVT(LRDef->getNode(), Reg, TII);
581  const TargetRegisterClass *RC =
582  TRI->getMinimalPhysRegClass(Reg, VT);
583  const TargetRegisterClass *DestRC = TRI->getCrossCopyRegClass(RC);
584 
585  // If cross copy register class is the same as RC, then it must be
586  // possible copy the value directly. Do not try duplicate the def.
587  // If cross copy register class is not the same as RC, then it's
588  // possible to copy the value but it require cross register class copies
589  // and it is expensive.
590  // If cross copy register class is null, then it's not possible to copy
591  // the value at all.
592  SUnit *NewDef = nullptr;
593  if (DestRC != RC) {
594  NewDef = CopyAndMoveSuccessors(LRDef);
595  if (!DestRC && !NewDef)
596  report_fatal_error("Can't handle live physical "
597  "register dependency!");
598  }
599  if (!NewDef) {
600  // Issue copies, these can be expensive cross register class copies.
602  InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
603  DEBUG(dbgs() << "Adding an edge from SU # " << TrySU->NodeNum
604  << " to SU #" << Copies.front()->NodeNum << "\n");
605  AddPred(TrySU, SDep(Copies.front(), SDep::Artificial));
606  NewDef = Copies.back();
607  }
608 
609  DEBUG(dbgs() << "Adding an edge from SU # " << NewDef->NodeNum
610  << " to SU #" << TrySU->NodeNum << "\n");
611  LiveRegDefs[Reg] = NewDef;
612  AddPred(NewDef, SDep(TrySU, SDep::Artificial));
613  TrySU->isAvailable = false;
614  CurSU = NewDef;
615  }
616 
617  if (!CurSU) {
618  llvm_unreachable("Unable to resolve live physical register dependencies!");
619  }
620  }
621 
622  // Add the nodes that aren't ready back onto the available list.
623  for (unsigned i = 0, e = NotReady.size(); i != e; ++i) {
624  NotReady[i]->isPending = false;
625  // May no longer be available due to backtracking.
626  if (NotReady[i]->isAvailable)
627  AvailableQueue.push(NotReady[i]);
628  }
629  NotReady.clear();
630 
631  if (CurSU)
632  ScheduleNodeBottomUp(CurSU, CurCycle);
633  ++CurCycle;
634  }
635 
636  // Reverse the order since it is bottom up.
637  std::reverse(Sequence.begin(), Sequence.end());
638 
639 #ifndef NDEBUG
640  VerifyScheduledSequence(/*isBottomUp=*/true);
641 #endif
642 }
643 
644 
645 namespace {
646 //===----------------------------------------------------------------------===//
647 // ScheduleDAGLinearize - No scheduling scheduler, it simply linearize the
648 // DAG in topological order.
649 // IMPORTANT: this may not work for targets with phyreg dependency.
650 //
651 class ScheduleDAGLinearize : public ScheduleDAGSDNodes {
652 public:
653  ScheduleDAGLinearize(MachineFunction &mf) : ScheduleDAGSDNodes(mf) {}
654 
655  void Schedule() override;
656 
658  EmitSchedule(MachineBasicBlock::iterator &InsertPos) override;
659 
660 private:
661  std::vector<SDNode*> Sequence;
662  DenseMap<SDNode*, SDNode*> GluedMap; // Cache glue to its user
663 
664  void ScheduleNode(SDNode *N);
665 };
666 } // end anonymous namespace
667 
668 void ScheduleDAGLinearize::ScheduleNode(SDNode *N) {
669  if (N->getNodeId() != 0)
670  llvm_unreachable(nullptr);
671 
672  if (!N->isMachineOpcode() &&
673  (N->getOpcode() == ISD::EntryToken || isPassiveNode(N)))
674  // These nodes do not need to be translated into MIs.
675  return;
676 
677  DEBUG(dbgs() << "\n*** Scheduling: ");
678  DEBUG(N->dump(DAG));
679  Sequence.push_back(N);
680 
681  unsigned NumOps = N->getNumOperands();
682  if (unsigned NumLeft = NumOps) {
683  SDNode *GluedOpN = nullptr;
684  do {
685  const SDValue &Op = N->getOperand(NumLeft-1);
686  SDNode *OpN = Op.getNode();
687 
688  if (NumLeft == NumOps && Op.getValueType() == MVT::Glue) {
689  // Schedule glue operand right above N.
690  GluedOpN = OpN;
691  assert(OpN->getNodeId() != 0 && "Glue operand not ready?");
692  OpN->setNodeId(0);
693  ScheduleNode(OpN);
694  continue;
695  }
696 
697  if (OpN == GluedOpN)
698  // Glue operand is already scheduled.
699  continue;
700 
701  DenseMap<SDNode*, SDNode*>::iterator DI = GluedMap.find(OpN);
702  if (DI != GluedMap.end() && DI->second != N)
703  // Users of glues are counted against the glued users.
704  OpN = DI->second;
705 
706  unsigned Degree = OpN->getNodeId();
707  assert(Degree > 0 && "Predecessor over-released!");
708  OpN->setNodeId(--Degree);
709  if (Degree == 0)
710  ScheduleNode(OpN);
711  } while (--NumLeft);
712  }
713 }
714 
715 /// findGluedUser - Find the representative use of a glue value by walking
716 /// the use chain.
718  while (SDNode *Glued = N->getGluedUser())
719  N = Glued;
720  return N;
721 }
722 
723 void ScheduleDAGLinearize::Schedule() {
724  DEBUG(dbgs() << "********** DAG Linearization **********\n");
725 
727  unsigned DAGSize = 0;
728  for (SDNode &Node : DAG->allnodes()) {
729  SDNode *N = &Node;
730 
731  // Use node id to record degree.
732  unsigned Degree = N->use_size();
733  N->setNodeId(Degree);
734  unsigned NumVals = N->getNumValues();
735  if (NumVals && N->getValueType(NumVals-1) == MVT::Glue &&
736  N->hasAnyUseOfValue(NumVals-1)) {
737  SDNode *User = findGluedUser(N);
738  if (User) {
739  Glues.push_back(N);
740  GluedMap.insert(std::make_pair(N, User));
741  }
742  }
743 
744  if (N->isMachineOpcode() ||
745  (N->getOpcode() != ISD::EntryToken && !isPassiveNode(N)))
746  ++DAGSize;
747  }
748 
749  for (unsigned i = 0, e = Glues.size(); i != e; ++i) {
750  SDNode *Glue = Glues[i];
751  SDNode *GUser = GluedMap[Glue];
752  unsigned Degree = Glue->getNodeId();
753  unsigned UDegree = GUser->getNodeId();
754 
755  // Glue user must be scheduled together with the glue operand. So other
756  // users of the glue operand must be treated as its users.
757  SDNode *ImmGUser = Glue->getGluedUser();
758  for (SDNode::use_iterator ui = Glue->use_begin(), ue = Glue->use_end();
759  ui != ue; ++ui)
760  if (*ui == ImmGUser)
761  --Degree;
762  GUser->setNodeId(UDegree + Degree);
763  Glue->setNodeId(1);
764  }
765 
766  Sequence.reserve(DAGSize);
767  ScheduleNode(DAG->getRoot().getNode());
768 }
769 
771 ScheduleDAGLinearize::EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
772  InstrEmitter Emitter(BB, InsertPos);
773  DenseMap<SDValue, unsigned> VRBaseMap;
774 
775  DEBUG({
776  dbgs() << "\n*** Final schedule ***\n";
777  });
778 
779  // FIXME: Handle dbg_values.
780  unsigned NumNodes = Sequence.size();
781  for (unsigned i = 0; i != NumNodes; ++i) {
782  SDNode *N = Sequence[NumNodes-i-1];
783  DEBUG(N->dump(DAG));
784  Emitter.EmitNode(N, false, false, VRBaseMap);
785  }
786 
787  DEBUG(dbgs() << '\n');
788 
789  InsertPos = Emitter.getInsertPos();
790  return Emitter.getBlock();
791 }
792 
793 //===----------------------------------------------------------------------===//
794 // Public Constructor Functions
795 //===----------------------------------------------------------------------===//
796 
799  return new ScheduleDAGFast(*IS->MF);
800 }
801 
804  return new ScheduleDAGLinearize(*IS->MF);
805 }
const uint16_t * getImplicitDefs() const
Return a list of registers that are potentially written by any instance of this machine instruction...
Definition: MCInstrDesc.h:497
void dump() const
Dump this node, for debugging.
STATISTIC(NumFunctions,"Total number of functions")
bool isCommutable() const
Return true if this may be a 2- or 3-address instruction (of the form "X = op Y, Z, ..."), which produces the same result if Y and Z are exchanged.
Definition: MCInstrDesc.h:388
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:191
bool isTwoAddress
Definition: ScheduleDAG.h:296
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:138
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
static bool isClobberKind(unsigned Flag)
Definition: InlineAsm.h:321
void setSUnit(SUnit *SU)
Definition: ScheduleDAG.h:165
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
void removePred(const SDep &D)
removePred - This removes the specified edge as a pred of the current node if it exists.
void setNodeId(int Id)
Set unique node id.
const TargetRegisterClass * CopyDstRC
Definition: ScheduleDAG.h:320
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:45
MachineFunction * MF
SmallVector< SDep, 4 > Preds
Definition: ScheduleDAG.h:275
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, bool gen_crash_diag=true)
Reports a serious error, calling any installed error handler.
bool isScheduled
Definition: ScheduleDAG.h:303
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:169
unsigned getHeight() const
getHeight - Return the height of this node, which is the length of the maximum path down to any node ...
Definition: ScheduleDAG.h:431
unsigned NumSuccsLeft
Definition: ScheduleDAG.h:288
const HexagonInstrInfo * TII
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
Regular data dependence (aka true-dependence).
Definition: ScheduleDAG.h:49
Reg
All possible values of the reg field in the ModR/M byte.
static bool isRegDefEarlyClobberKind(unsigned Flag)
Definition: InlineAsm.h:318
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:571
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
ScheduleDAGSDNodes * createDAGLinearizer(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDAGLinearizer - This creates a "no-scheduling" scheduler which linearize the DAG using topologi...
bool isPending
Definition: ScheduleDAG.h:301
static bool isRegDefKind(unsigned Flag)
Definition: InlineAsm.h:315
TargetInstrInfo - Interface to description of machine instruction set.
SDep - Scheduling dependency.
Definition: ScheduleDAG.h:45
SDNode * getNode() const
get the SDNode which holds the desired result
bundle_iterator< MachineInstr, instr_iterator > iterator
void setHeightToAtLeast(unsigned NewHeight)
setDepthToAtLeast - If NewDepth is greater than this node's depth value, set it to be the new height ...
bool isAvailable()
Definition: Compression.cpp:48
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
size_t use_size() const
Return the number of uses of this node.
MVT - Machine Value Type.
SDNode * getNode() const
getNode - Return the representative SDNode for this SUnit.
Definition: ScheduleDAG.h:388
unsigned short Latency
Definition: ScheduleDAG.h:292
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:32
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
Definition: InlineAsm.h:332
MCRegAliasIterator enumerates all registers aliasing Reg.
This class provides iterator support for SDUse operands that use a specific SDNode.
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:69
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specific constraint if it is set.
Definition: MCInstrDesc.h:162
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
An unknown scheduling barrier.
Definition: ScheduleDAG.h:65
bool isCommutable
Definition: ScheduleDAG.h:297
static MVT getPhysicalRegisterVT(SDNode *N, unsigned Reg, const TargetInstrInfo *TII)
getPhysicalRegisterVT - Returns the ValueType of the physical register definition of the specified no...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
SDNode * getGluedUser() const
If this node has a glue value with a user, return the user (there is at most one).
SI Lower i1 Copies
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
const TargetRegisterClass * CopySrcRC
Definition: ScheduleDAG.h:321
static SDNode * findGluedUser(SDNode *N)
findGluedUser - Find the representative use of a glue value by walking the use chain.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
static RegisterScheduler fastDAGScheduler("fast","Fast suboptimal list scheduling", createFastDAGScheduler)
SUnit * getSUnit() const
Definition: ScheduleDAG.h:160
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
static use_iterator use_end()
iterator_range< value_op_iterator > op_values() const
void setLatency(unsigned Lat)
setLatency - Set the latency for this edge.
Definition: ScheduleDAG.h:155
bool isAvailable
Definition: ScheduleDAG.h:302
static bool isPhysicalRegister(unsigned Reg)
isPhysicalRegister - Return true if the specified register number is in the physical register namespa...
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
Sequence
A sequence of states that a pointer may go through in which an objc_retain and objc_release are actua...
Definition: PtrState.h:37
static bool CheckForLiveRegDef(SUnit *SU, unsigned Reg, std::vector< SUnit * > &LiveRegDefs, SmallSet< unsigned, 4 > &RegAdded, SmallVectorImpl< unsigned > &LRegs, const TargetRegisterInfo *TRI)
CheckForLiveRegDef - Return true and update live register vector if the specified register def of the...
static RegisterScheduler linearizeDAGScheduler("linearize","Linearize DAG, no scheduling", createDAGLinearizer)
int getNodeId() const
Return the unique node id.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:166
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned NodeNum
Definition: ScheduleDAG.h:283
const uint16_t * ImplicitDefs
Definition: MCInstrDesc.h:148
bool addPred(const SDep &D, bool Required=true)
addPred - This adds the specified edge as a pred of the current node if not already.
Definition: ScheduleDAG.cpp:65
SmallVector< SDep, 4 > Succs
Definition: ScheduleDAG.h:276
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:185
Arbitrary strong DAG edge (no real dependence).
Definition: ScheduleDAG.h:68
ScheduleDAGSDNodes * createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createFastDAGScheduler - This creates a "fast" scheduler.
#define DEBUG(X)
Definition: Debug.h:92
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
void dump(const ScheduleDAG *G) const
SUnit - Scheduling unit.
SUnit - Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:261
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.