LLVM  4.0.0
ScheduleDAGFast.cpp
Go to the documentation of this file.
1 //===----- ScheduleDAGFast.cpp - Fast poor list scheduler -----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements a fast scheduler.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "InstrEmitter.h"
16 #include "ScheduleDAGSDNodes.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/Statistic.h"
21 #include "llvm/IR/DataLayout.h"
22 #include "llvm/IR/InlineAsm.h"
23 #include "llvm/Support/Debug.h"
28 using namespace llvm;
29 
30 #define DEBUG_TYPE "pre-RA-sched"
31 
32 STATISTIC(NumUnfolds, "Number of nodes unfolded");
33 STATISTIC(NumDups, "Number of duplicated nodes");
34 STATISTIC(NumPRCopies, "Number of physical copies");
35 
36 static RegisterScheduler
37  fastDAGScheduler("fast", "Fast suboptimal list scheduling",
39 static RegisterScheduler
40  linearizeDAGScheduler("linearize", "Linearize DAG, no scheduling",
42 
43 
44 namespace {
45  /// FastPriorityQueue - A degenerate priority queue that considers
46  /// all nodes to have the same priority.
47  ///
48  struct FastPriorityQueue {
50 
51  bool empty() const { return Queue.empty(); }
52 
53  void push(SUnit *U) {
54  Queue.push_back(U);
55  }
56 
57  SUnit *pop() {
58  if (empty()) return nullptr;
59  SUnit *V = Queue.back();
60  Queue.pop_back();
61  return V;
62  }
63  };
64 
65 //===----------------------------------------------------------------------===//
66 /// ScheduleDAGFast - The actual "fast" list scheduler implementation.
67 ///
68 class ScheduleDAGFast : public ScheduleDAGSDNodes {
69 private:
70  /// AvailableQueue - The priority queue to use for the available SUnits.
71  FastPriorityQueue AvailableQueue;
72 
73  /// LiveRegDefs - A set of physical registers and their definition
74  /// that are "live". These nodes must be scheduled before any other nodes that
75  /// modifies the registers can be scheduled.
76  unsigned NumLiveRegs;
77  std::vector<SUnit*> LiveRegDefs;
78  std::vector<unsigned> LiveRegCycles;
79 
80 public:
81  ScheduleDAGFast(MachineFunction &mf)
82  : ScheduleDAGSDNodes(mf) {}
83 
84  void Schedule() override;
85 
86  /// AddPred - adds a predecessor edge to SUnit SU.
87  /// This returns true if this is a new predecessor.
88  void AddPred(SUnit *SU, const SDep &D) {
89  SU->addPred(D);
90  }
91 
92  /// RemovePred - removes a predecessor edge from SUnit SU.
93  /// This returns true if an edge was removed.
94  void RemovePred(SUnit *SU, const SDep &D) {
95  SU->removePred(D);
96  }
97 
98 private:
99  void ReleasePred(SUnit *SU, SDep *PredEdge);
100  void ReleasePredecessors(SUnit *SU, unsigned CurCycle);
101  void ScheduleNodeBottomUp(SUnit*, unsigned);
102  SUnit *CopyAndMoveSuccessors(SUnit*);
103  void InsertCopiesAndMoveSuccs(SUnit*, unsigned,
104  const TargetRegisterClass*,
105  const TargetRegisterClass*,
107  bool DelayForLiveRegsBottomUp(SUnit*, SmallVectorImpl<unsigned>&);
108  void ListScheduleBottomUp();
109 
110  /// forceUnitLatencies - The fast scheduler doesn't care about real latencies.
111  bool forceUnitLatencies() const override { return true; }
112 };
113 } // end anonymous namespace
114 
115 
116 /// Schedule - Schedule the DAG using list scheduling.
117 void ScheduleDAGFast::Schedule() {
118  DEBUG(dbgs() << "********** List Scheduling **********\n");
119 
120  NumLiveRegs = 0;
121  LiveRegDefs.resize(TRI->getNumRegs(), nullptr);
122  LiveRegCycles.resize(TRI->getNumRegs(), 0);
123 
124  // Build the scheduling graph.
125  BuildSchedGraph(nullptr);
126 
127  DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
128  SUnits[su].dumpAll(this));
129 
130  // Execute the actual scheduling loop.
131  ListScheduleBottomUp();
132 }
133 
134 //===----------------------------------------------------------------------===//
135 // Bottom-Up Scheduling
136 //===----------------------------------------------------------------------===//
137 
138 /// ReleasePred - Decrement the NumSuccsLeft count of a predecessor. Add it to
139 /// the AvailableQueue if the count reaches zero. Also update its cycle bound.
140 void ScheduleDAGFast::ReleasePred(SUnit *SU, SDep *PredEdge) {
141  SUnit *PredSU = PredEdge->getSUnit();
142 
143 #ifndef NDEBUG
144  if (PredSU->NumSuccsLeft == 0) {
145  dbgs() << "*** Scheduling failed! ***\n";
146  PredSU->dump(this);
147  dbgs() << " has been released too many times!\n";
148  llvm_unreachable(nullptr);
149  }
150 #endif
151  --PredSU->NumSuccsLeft;
152 
153  // If all the node's successors are scheduled, this node is ready
154  // to be scheduled. Ignore the special EntrySU node.
155  if (PredSU->NumSuccsLeft == 0 && PredSU != &EntrySU) {
156  PredSU->isAvailable = true;
157  AvailableQueue.push(PredSU);
158  }
159 }
160 
161 void ScheduleDAGFast::ReleasePredecessors(SUnit *SU, unsigned CurCycle) {
162  // Bottom up: release predecessors
163  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
164  I != E; ++I) {
165  ReleasePred(SU, &*I);
166  if (I->isAssignedRegDep()) {
167  // This is a physical register dependency and it's impossible or
168  // expensive to copy the register. Make sure nothing that can
169  // clobber the register is scheduled between the predecessor and
170  // this node.
171  if (!LiveRegDefs[I->getReg()]) {
172  ++NumLiveRegs;
173  LiveRegDefs[I->getReg()] = I->getSUnit();
174  LiveRegCycles[I->getReg()] = CurCycle;
175  }
176  }
177  }
178 }
179 
180 /// ScheduleNodeBottomUp - Add the node to the schedule. Decrement the pending
181 /// count of its predecessors. If a predecessor pending count is zero, add it to
182 /// the Available queue.
183 void ScheduleDAGFast::ScheduleNodeBottomUp(SUnit *SU, unsigned CurCycle) {
184  DEBUG(dbgs() << "*** Scheduling [" << CurCycle << "]: ");
185  DEBUG(SU->dump(this));
186 
187  assert(CurCycle >= SU->getHeight() && "Node scheduled below its height!");
188  SU->setHeightToAtLeast(CurCycle);
189  Sequence.push_back(SU);
190 
191  ReleasePredecessors(SU, CurCycle);
192 
193  // Release all the implicit physical register defs that are live.
194  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
195  I != E; ++I) {
196  if (I->isAssignedRegDep()) {
197  if (LiveRegCycles[I->getReg()] == I->getSUnit()->getHeight()) {
198  assert(NumLiveRegs > 0 && "NumLiveRegs is already zero!");
199  assert(LiveRegDefs[I->getReg()] == SU &&
200  "Physical register dependency violated?");
201  --NumLiveRegs;
202  LiveRegDefs[I->getReg()] = nullptr;
203  LiveRegCycles[I->getReg()] = 0;
204  }
205  }
206  }
207 
208  SU->isScheduled = true;
209 }
210 
211 /// CopyAndMoveSuccessors - Clone the specified node and move its scheduled
212 /// successors to the newly created node.
213 SUnit *ScheduleDAGFast::CopyAndMoveSuccessors(SUnit *SU) {
214  if (SU->getNode()->getGluedNode())
215  return nullptr;
216 
217  SDNode *N = SU->getNode();
218  if (!N)
219  return nullptr;
220 
221  SUnit *NewSU;
222  bool TryUnfold = false;
223  for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
224  MVT VT = N->getSimpleValueType(i);
225  if (VT == MVT::Glue)
226  return nullptr;
227  else if (VT == MVT::Other)
228  TryUnfold = true;
229  }
230  for (const SDValue &Op : N->op_values()) {
231  MVT VT = Op.getNode()->getSimpleValueType(Op.getResNo());
232  if (VT == MVT::Glue)
233  return nullptr;
234  }
235 
236  if (TryUnfold) {
237  SmallVector<SDNode*, 2> NewNodes;
238  if (!TII->unfoldMemoryOperand(*DAG, N, NewNodes))
239  return nullptr;
240 
241  DEBUG(dbgs() << "Unfolding SU # " << SU->NodeNum << "\n");
242  assert(NewNodes.size() == 2 && "Expected a load folding node!");
243 
244  N = NewNodes[1];
245  SDNode *LoadNode = NewNodes[0];
246  unsigned NumVals = N->getNumValues();
247  unsigned OldNumVals = SU->getNode()->getNumValues();
248  for (unsigned i = 0; i != NumVals; ++i)
249  DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i));
250  DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals-1),
251  SDValue(LoadNode, 1));
252 
253  SUnit *NewSU = newSUnit(N);
254  assert(N->getNodeId() == -1 && "Node already inserted!");
255  N->setNodeId(NewSU->NodeNum);
256 
257  const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
258  for (unsigned i = 0; i != MCID.getNumOperands(); ++i) {
259  if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) {
260  NewSU->isTwoAddress = true;
261  break;
262  }
263  }
264  if (MCID.isCommutable())
265  NewSU->isCommutable = true;
266 
267  // LoadNode may already exist. This can happen when there is another
268  // load from the same location and producing the same type of value
269  // but it has different alignment or volatileness.
270  bool isNewLoad = true;
271  SUnit *LoadSU;
272  if (LoadNode->getNodeId() != -1) {
273  LoadSU = &SUnits[LoadNode->getNodeId()];
274  isNewLoad = false;
275  } else {
276  LoadSU = newSUnit(LoadNode);
277  LoadNode->setNodeId(LoadSU->NodeNum);
278  }
279 
280  SDep ChainPred;
281  SmallVector<SDep, 4> ChainSuccs;
282  SmallVector<SDep, 4> LoadPreds;
283  SmallVector<SDep, 4> NodePreds;
284  SmallVector<SDep, 4> NodeSuccs;
285  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
286  I != E; ++I) {
287  if (I->isCtrl())
288  ChainPred = *I;
289  else if (I->getSUnit()->getNode() &&
290  I->getSUnit()->getNode()->isOperandOf(LoadNode))
291  LoadPreds.push_back(*I);
292  else
293  NodePreds.push_back(*I);
294  }
295  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
296  I != E; ++I) {
297  if (I->isCtrl())
298  ChainSuccs.push_back(*I);
299  else
300  NodeSuccs.push_back(*I);
301  }
302 
303  if (ChainPred.getSUnit()) {
304  RemovePred(SU, ChainPred);
305  if (isNewLoad)
306  AddPred(LoadSU, ChainPred);
307  }
308  for (unsigned i = 0, e = LoadPreds.size(); i != e; ++i) {
309  const SDep &Pred = LoadPreds[i];
310  RemovePred(SU, Pred);
311  if (isNewLoad) {
312  AddPred(LoadSU, Pred);
313  }
314  }
315  for (unsigned i = 0, e = NodePreds.size(); i != e; ++i) {
316  const SDep &Pred = NodePreds[i];
317  RemovePred(SU, Pred);
318  AddPred(NewSU, Pred);
319  }
320  for (unsigned i = 0, e = NodeSuccs.size(); i != e; ++i) {
321  SDep D = NodeSuccs[i];
322  SUnit *SuccDep = D.getSUnit();
323  D.setSUnit(SU);
324  RemovePred(SuccDep, D);
325  D.setSUnit(NewSU);
326  AddPred(SuccDep, D);
327  }
328  for (unsigned i = 0, e = ChainSuccs.size(); i != e; ++i) {
329  SDep D = ChainSuccs[i];
330  SUnit *SuccDep = D.getSUnit();
331  D.setSUnit(SU);
332  RemovePred(SuccDep, D);
333  if (isNewLoad) {
334  D.setSUnit(LoadSU);
335  AddPred(SuccDep, D);
336  }
337  }
338  if (isNewLoad) {
339  SDep D(LoadSU, SDep::Barrier);
340  D.setLatency(LoadSU->Latency);
341  AddPred(NewSU, D);
342  }
343 
344  ++NumUnfolds;
345 
346  if (NewSU->NumSuccsLeft == 0) {
347  NewSU->isAvailable = true;
348  return NewSU;
349  }
350  SU = NewSU;
351  }
352 
353  DEBUG(dbgs() << "Duplicating SU # " << SU->NodeNum << "\n");
354  NewSU = Clone(SU);
355 
356  // New SUnit has the exact same predecessors.
357  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
358  I != E; ++I)
359  if (!I->isArtificial())
360  AddPred(NewSU, *I);
361 
362  // Only copy scheduled successors. Cut them from old node's successor
363  // list and move them over.
365  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
366  I != E; ++I) {
367  if (I->isArtificial())
368  continue;
369  SUnit *SuccSU = I->getSUnit();
370  if (SuccSU->isScheduled) {
371  SDep D = *I;
372  D.setSUnit(NewSU);
373  AddPred(SuccSU, D);
374  D.setSUnit(SU);
375  DelDeps.push_back(std::make_pair(SuccSU, D));
376  }
377  }
378  for (unsigned i = 0, e = DelDeps.size(); i != e; ++i)
379  RemovePred(DelDeps[i].first, DelDeps[i].second);
380 
381  ++NumDups;
382  return NewSU;
383 }
384 
385 /// InsertCopiesAndMoveSuccs - Insert register copies and move all
386 /// scheduled successors of the given SUnit to the last copy.
387 void ScheduleDAGFast::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg,
388  const TargetRegisterClass *DestRC,
389  const TargetRegisterClass *SrcRC,
390  SmallVectorImpl<SUnit*> &Copies) {
391  SUnit *CopyFromSU = newSUnit(static_cast<SDNode *>(nullptr));
392  CopyFromSU->CopySrcRC = SrcRC;
393  CopyFromSU->CopyDstRC = DestRC;
394 
395  SUnit *CopyToSU = newSUnit(static_cast<SDNode *>(nullptr));
396  CopyToSU->CopySrcRC = DestRC;
397  CopyToSU->CopyDstRC = SrcRC;
398 
399  // Only copy scheduled successors. Cut them from old node's successor
400  // list and move them over.
402  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
403  I != E; ++I) {
404  if (I->isArtificial())
405  continue;
406  SUnit *SuccSU = I->getSUnit();
407  if (SuccSU->isScheduled) {
408  SDep D = *I;
409  D.setSUnit(CopyToSU);
410  AddPred(SuccSU, D);
411  DelDeps.push_back(std::make_pair(SuccSU, *I));
412  }
413  }
414  for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) {
415  RemovePred(DelDeps[i].first, DelDeps[i].second);
416  }
417  SDep FromDep(SU, SDep::Data, Reg);
418  FromDep.setLatency(SU->Latency);
419  AddPred(CopyFromSU, FromDep);
420  SDep ToDep(CopyFromSU, SDep::Data, 0);
421  ToDep.setLatency(CopyFromSU->Latency);
422  AddPred(CopyToSU, ToDep);
423 
424  Copies.push_back(CopyFromSU);
425  Copies.push_back(CopyToSU);
426 
427  ++NumPRCopies;
428 }
429 
430 /// getPhysicalRegisterVT - Returns the ValueType of the physical register
431 /// definition of the specified node.
432 /// FIXME: Move to SelectionDAG?
433 static MVT getPhysicalRegisterVT(SDNode *N, unsigned Reg,
434  const TargetInstrInfo *TII) {
435  unsigned NumRes;
436  if (N->getOpcode() == ISD::CopyFromReg) {
437  // CopyFromReg has: "chain, Val, glue" so operand 1 gives the type.
438  NumRes = 1;
439  } else {
440  const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
441  assert(MCID.ImplicitDefs && "Physical reg def must be in implicit def list!");
442  NumRes = MCID.getNumDefs();
443  for (const MCPhysReg *ImpDef = MCID.getImplicitDefs(); *ImpDef; ++ImpDef) {
444  if (Reg == *ImpDef)
445  break;
446  ++NumRes;
447  }
448  }
449  return N->getSimpleValueType(NumRes);
450 }
451 
452 /// CheckForLiveRegDef - Return true and update live register vector if the
453 /// specified register def of the specified SUnit clobbers any "live" registers.
454 static bool CheckForLiveRegDef(SUnit *SU, unsigned Reg,
455  std::vector<SUnit*> &LiveRegDefs,
456  SmallSet<unsigned, 4> &RegAdded,
458  const TargetRegisterInfo *TRI) {
459  bool Added = false;
460  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) {
461  if (LiveRegDefs[*AI] && LiveRegDefs[*AI] != SU) {
462  if (RegAdded.insert(*AI).second) {
463  LRegs.push_back(*AI);
464  Added = true;
465  }
466  }
467  }
468  return Added;
469 }
470 
471 /// DelayForLiveRegsBottomUp - Returns true if it is necessary to delay
472 /// scheduling of the given node to satisfy live physical register dependencies.
473 /// If the specific node is the last one that's available to schedule, do
474 /// whatever is necessary (i.e. backtracking or cloning) to make it possible.
475 bool ScheduleDAGFast::DelayForLiveRegsBottomUp(SUnit *SU,
477  if (NumLiveRegs == 0)
478  return false;
479 
480  SmallSet<unsigned, 4> RegAdded;
481  // If this node would clobber any "live" register, then it's not ready.
482  for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
483  I != E; ++I) {
484  if (I->isAssignedRegDep()) {
485  CheckForLiveRegDef(I->getSUnit(), I->getReg(), LiveRegDefs,
486  RegAdded, LRegs, TRI);
487  }
488  }
489 
490  for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) {
491  if (Node->getOpcode() == ISD::INLINEASM) {
492  // Inline asm can clobber physical defs.
493  unsigned NumOps = Node->getNumOperands();
494  if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
495  --NumOps; // Ignore the glue operand.
496 
497  for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
498  unsigned Flags =
499  cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
500  unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
501 
502  ++i; // Skip the ID value.
503  if (InlineAsm::isRegDefKind(Flags) ||
505  InlineAsm::isClobberKind(Flags)) {
506  // Check for def of register or earlyclobber register.
507  for (; NumVals; --NumVals, ++i) {
508  unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
510  CheckForLiveRegDef(SU, Reg, LiveRegDefs, RegAdded, LRegs, TRI);
511  }
512  } else
513  i += NumVals;
514  }
515  continue;
516  }
517  if (!Node->isMachineOpcode())
518  continue;
519  const MCInstrDesc &MCID = TII->get(Node->getMachineOpcode());
520  if (!MCID.ImplicitDefs)
521  continue;
522  for (const MCPhysReg *Reg = MCID.getImplicitDefs(); *Reg; ++Reg) {
523  CheckForLiveRegDef(SU, *Reg, LiveRegDefs, RegAdded, LRegs, TRI);
524  }
525  }
526  return !LRegs.empty();
527 }
528 
529 
530 /// ListScheduleBottomUp - The main loop of list scheduling for bottom-up
531 /// schedulers.
532 void ScheduleDAGFast::ListScheduleBottomUp() {
533  unsigned CurCycle = 0;
534 
535  // Release any predecessors of the special Exit node.
536  ReleasePredecessors(&ExitSU, CurCycle);
537 
538  // Add root to Available queue.
539  if (!SUnits.empty()) {
540  SUnit *RootSU = &SUnits[DAG->getRoot().getNode()->getNodeId()];
541  assert(RootSU->Succs.empty() && "Graph root shouldn't have successors!");
542  RootSU->isAvailable = true;
543  AvailableQueue.push(RootSU);
544  }
545 
546  // While Available queue is not empty, grab the node with the highest
547  // priority. If it is not ready put it back. Schedule the node.
548  SmallVector<SUnit*, 4> NotReady;
550  Sequence.reserve(SUnits.size());
551  while (!AvailableQueue.empty()) {
552  bool Delayed = false;
553  LRegsMap.clear();
554  SUnit *CurSU = AvailableQueue.pop();
555  while (CurSU) {
557  if (!DelayForLiveRegsBottomUp(CurSU, LRegs))
558  break;
559  Delayed = true;
560  LRegsMap.insert(std::make_pair(CurSU, LRegs));
561 
562  CurSU->isPending = true; // This SU is not in AvailableQueue right now.
563  NotReady.push_back(CurSU);
564  CurSU = AvailableQueue.pop();
565  }
566 
567  // All candidates are delayed due to live physical reg dependencies.
568  // Try code duplication or inserting cross class copies
569  // to resolve it.
570  if (Delayed && !CurSU) {
571  if (!CurSU) {
572  // Try duplicating the nodes that produces these
573  // "expensive to copy" values to break the dependency. In case even
574  // that doesn't work, insert cross class copies.
575  SUnit *TrySU = NotReady[0];
576  SmallVectorImpl<unsigned> &LRegs = LRegsMap[TrySU];
577  assert(LRegs.size() == 1 && "Can't handle this yet!");
578  unsigned Reg = LRegs[0];
579  SUnit *LRDef = LiveRegDefs[Reg];
580  MVT VT = getPhysicalRegisterVT(LRDef->getNode(), Reg, TII);
581  const TargetRegisterClass *RC =
582  TRI->getMinimalPhysRegClass(Reg, VT);
583  const TargetRegisterClass *DestRC = TRI->getCrossCopyRegClass(RC);
584 
585  // If cross copy register class is the same as RC, then it must be
586  // possible copy the value directly. Do not try duplicate the def.
587  // If cross copy register class is not the same as RC, then it's
588  // possible to copy the value but it require cross register class copies
589  // and it is expensive.
590  // If cross copy register class is null, then it's not possible to copy
591  // the value at all.
592  SUnit *NewDef = nullptr;
593  if (DestRC != RC) {
594  NewDef = CopyAndMoveSuccessors(LRDef);
595  if (!DestRC && !NewDef)
596  report_fatal_error("Can't handle live physical "
597  "register dependency!");
598  }
599  if (!NewDef) {
600  // Issue copies, these can be expensive cross register class copies.
601  SmallVector<SUnit*, 2> Copies;
602  InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
603  DEBUG(dbgs() << "Adding an edge from SU # " << TrySU->NodeNum
604  << " to SU #" << Copies.front()->NodeNum << "\n");
605  AddPred(TrySU, SDep(Copies.front(), SDep::Artificial));
606  NewDef = Copies.back();
607  }
608 
609  DEBUG(dbgs() << "Adding an edge from SU # " << NewDef->NodeNum
610  << " to SU #" << TrySU->NodeNum << "\n");
611  LiveRegDefs[Reg] = NewDef;
612  AddPred(NewDef, SDep(TrySU, SDep::Artificial));
613  TrySU->isAvailable = false;
614  CurSU = NewDef;
615  }
616 
617  if (!CurSU) {
618  llvm_unreachable("Unable to resolve live physical register dependencies!");
619  }
620  }
621 
622  // Add the nodes that aren't ready back onto the available list.
623  for (unsigned i = 0, e = NotReady.size(); i != e; ++i) {
624  NotReady[i]->isPending = false;
625  // May no longer be available due to backtracking.
626  if (NotReady[i]->isAvailable)
627  AvailableQueue.push(NotReady[i]);
628  }
629  NotReady.clear();
630 
631  if (CurSU)
632  ScheduleNodeBottomUp(CurSU, CurCycle);
633  ++CurCycle;
634  }
635 
636  // Reverse the order since it is bottom up.
637  std::reverse(Sequence.begin(), Sequence.end());
638 
639 #ifndef NDEBUG
640  VerifyScheduledSequence(/*isBottomUp=*/true);
641 #endif
642 }
643 
644 
645 namespace {
646 //===----------------------------------------------------------------------===//
647 // ScheduleDAGLinearize - No scheduling scheduler, it simply linearize the
648 // DAG in topological order.
649 // IMPORTANT: this may not work for targets with phyreg dependency.
650 //
651 class ScheduleDAGLinearize : public ScheduleDAGSDNodes {
652 public:
653  ScheduleDAGLinearize(MachineFunction &mf) : ScheduleDAGSDNodes(mf) {}
654 
655  void Schedule() override;
656 
658  EmitSchedule(MachineBasicBlock::iterator &InsertPos) override;
659 
660 private:
661  std::vector<SDNode*> Sequence;
662  DenseMap<SDNode*, SDNode*> GluedMap; // Cache glue to its user
663 
664  void ScheduleNode(SDNode *N);
665 };
666 } // end anonymous namespace
667 
668 void ScheduleDAGLinearize::ScheduleNode(SDNode *N) {
669  if (N->getNodeId() != 0)
670  llvm_unreachable(nullptr);
671 
672  if (!N->isMachineOpcode() &&
673  (N->getOpcode() == ISD::EntryToken || isPassiveNode(N)))
674  // These nodes do not need to be translated into MIs.
675  return;
676 
677  DEBUG(dbgs() << "\n*** Scheduling: ");
678  DEBUG(N->dump(DAG));
679  Sequence.push_back(N);
680 
681  unsigned NumOps = N->getNumOperands();
682  if (unsigned NumLeft = NumOps) {
683  SDNode *GluedOpN = nullptr;
684  do {
685  const SDValue &Op = N->getOperand(NumLeft-1);
686  SDNode *OpN = Op.getNode();
687 
688  if (NumLeft == NumOps && Op.getValueType() == MVT::Glue) {
689  // Schedule glue operand right above N.
690  GluedOpN = OpN;
691  assert(OpN->getNodeId() != 0 && "Glue operand not ready?");
692  OpN->setNodeId(0);
693  ScheduleNode(OpN);
694  continue;
695  }
696 
697  if (OpN == GluedOpN)
698  // Glue operand is already scheduled.
699  continue;
700 
701  DenseMap<SDNode*, SDNode*>::iterator DI = GluedMap.find(OpN);
702  if (DI != GluedMap.end() && DI->second != N)
703  // Users of glues are counted against the glued users.
704  OpN = DI->second;
705 
706  unsigned Degree = OpN->getNodeId();
707  assert(Degree > 0 && "Predecessor over-released!");
708  OpN->setNodeId(--Degree);
709  if (Degree == 0)
710  ScheduleNode(OpN);
711  } while (--NumLeft);
712  }
713 }
714 
715 /// findGluedUser - Find the representative use of a glue value by walking
716 /// the use chain.
718  while (SDNode *Glued = N->getGluedUser())
719  N = Glued;
720  return N;
721 }
722 
723 void ScheduleDAGLinearize::Schedule() {
724  DEBUG(dbgs() << "********** DAG Linearization **********\n");
725 
727  unsigned DAGSize = 0;
728  for (SDNode &Node : DAG->allnodes()) {
729  SDNode *N = &Node;
730 
731  // Use node id to record degree.
732  unsigned Degree = N->use_size();
733  N->setNodeId(Degree);
734  unsigned NumVals = N->getNumValues();
735  if (NumVals && N->getValueType(NumVals-1) == MVT::Glue &&
736  N->hasAnyUseOfValue(NumVals-1)) {
737  SDNode *User = findGluedUser(N);
738  if (User) {
739  Glues.push_back(N);
740  GluedMap.insert(std::make_pair(N, User));
741  }
742  }
743 
744  if (N->isMachineOpcode() ||
745  (N->getOpcode() != ISD::EntryToken && !isPassiveNode(N)))
746  ++DAGSize;
747  }
748 
749  for (unsigned i = 0, e = Glues.size(); i != e; ++i) {
750  SDNode *Glue = Glues[i];
751  SDNode *GUser = GluedMap[Glue];
752  unsigned Degree = Glue->getNodeId();
753  unsigned UDegree = GUser->getNodeId();
754 
755  // Glue user must be scheduled together with the glue operand. So other
756  // users of the glue operand must be treated as its users.
757  SDNode *ImmGUser = Glue->getGluedUser();
758  for (SDNode::use_iterator ui = Glue->use_begin(), ue = Glue->use_end();
759  ui != ue; ++ui)
760  if (*ui == ImmGUser)
761  --Degree;
762  GUser->setNodeId(UDegree + Degree);
763  Glue->setNodeId(1);
764  }
765 
766  Sequence.reserve(DAGSize);
767  ScheduleNode(DAG->getRoot().getNode());
768 }
769 
771 ScheduleDAGLinearize::EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
772  InstrEmitter Emitter(BB, InsertPos);
773  DenseMap<SDValue, unsigned> VRBaseMap;
774 
775  DEBUG({
776  dbgs() << "\n*** Final schedule ***\n";
777  });
778 
779  // FIXME: Handle dbg_values.
780  unsigned NumNodes = Sequence.size();
781  for (unsigned i = 0; i != NumNodes; ++i) {
782  SDNode *N = Sequence[NumNodes-i-1];
783  DEBUG(N->dump(DAG));
784  Emitter.EmitNode(N, false, false, VRBaseMap);
785  }
786 
787  DEBUG(dbgs() << '\n');
788 
789  InsertPos = Emitter.getInsertPos();
790  return Emitter.getBlock();
791 }
792 
793 //===----------------------------------------------------------------------===//
794 // Public Constructor Functions
795 //===----------------------------------------------------------------------===//
796 
799  return new ScheduleDAGFast(*IS->MF);
800 }
801 
804  return new ScheduleDAGLinearize(*IS->MF);
805 }
STATISTIC(NumFunctions,"Total number of functions")
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
size_t i
bool isCommutable() const
Return true if this may be a 2- or 3-address instruction (of the form "X = op Y, Z, ..."), which produces the same result if Y and Z are exchanged.
Definition: MCInstrDesc.h:416
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:216
bool isTwoAddress
Definition: ScheduleDAG.h:279
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
static bool isClobberKind(unsigned Flag)
Definition: InlineAsm.h:280
void setSUnit(SUnit *SU)
Definition: ScheduleDAG.h:506
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
void removePred(const SDep &D)
removePred - This removes the specified edge as a pred of the current node if it exists.
void setNodeId(int Id)
Set unique node id.
const TargetRegisterClass * CopyDstRC
Definition: ScheduleDAG.h:303
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:45
MachineFunction * MF
SmallVector< SDep, 4 > Preds
Definition: ScheduleDAG.h:258
bool isScheduled
Definition: ScheduleDAG.h:286
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:172
unsigned getHeight() const
getHeight - Return the height of this node, which is the length of the maximum path down to any node ...
Definition: ScheduleDAG.h:425
unsigned NumSuccsLeft
Definition: ScheduleDAG.h:271
struct fuzzer::@269 Flags
const HexagonInstrInfo * TII
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
Regular data dependence (aka true-dependence).
Definition: ScheduleDAG.h:49
Reg
All possible values of the reg field in the ModR/M byte.
static bool isRegDefEarlyClobberKind(unsigned Flag)
Definition: InlineAsm.h:277
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:589
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:241
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
ScheduleDAGSDNodes * createDAGLinearizer(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createDAGLinearizer - This creates a "no-scheduling" scheduler which linearize the DAG using topologi...
const MCPhysReg * getImplicitDefs() const
Return a list of registers that are potentially written by any instance of this machine instruction...
Definition: MCInstrDesc.h:525
bool isPending
Definition: ScheduleDAG.h:284
static bool isRegDefKind(unsigned Flag)
Definition: InlineAsm.h:274
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
TargetInstrInfo - Interface to description of machine instruction set.
SDep - Scheduling dependency.
Definition: ScheduleDAG.h:45
SDNode * getNode() const
get the SDNode which holds the desired result
void setHeightToAtLeast(unsigned NewHeight)
setDepthToAtLeast - If NewDepth is greater than this node's depth value, set it to be the new height ...
bool isAvailable()
Definition: Compression.cpp:48
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
size_t use_size() const
Return the number of uses of this node.
MVT - Machine Value Type.
SDNode * getNode() const
getNode - Return the representative SDNode for this SUnit.
Definition: ScheduleDAG.h:371
unsigned short Latency
Definition: ScheduleDAG.h:275
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:36
const MCPhysReg * ImplicitDefs
Definition: MCInstrDesc.h:173
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
Definition: InlineAsm.h:335
MCRegAliasIterator enumerates all registers aliasing Reg.
This class provides iterator support for SDUse operands that use a specific SDNode.
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:80
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specific constraint if it is set.
Definition: MCInstrDesc.h:187
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
An unknown scheduling barrier.
Definition: ScheduleDAG.h:65
bool isCommutable
Definition: ScheduleDAG.h:280
static MVT getPhysicalRegisterVT(SDNode *N, unsigned Reg, const TargetInstrInfo *TII)
getPhysicalRegisterVT - Returns the ValueType of the physical register definition of the specified no...
void dump() const
Dump this node, for debugging.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
SDNode * getGluedUser() const
If this node has a glue value with a user, return the user (there is at most one).
Represents one node in the SelectionDAG.
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
const TargetRegisterClass * CopySrcRC
Definition: ScheduleDAG.h:304
static SDNode * findGluedUser(SDNode *N)
findGluedUser - Find the representative use of a glue value by walking the use chain.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
static RegisterScheduler fastDAGScheduler("fast","Fast suboptimal list scheduling", createFastDAGScheduler)
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
static use_iterator use_end()
iterator_range< value_op_iterator > op_values() const
void setLatency(unsigned Lat)
setLatency - Set the latency for this edge.
Definition: ScheduleDAG.h:144
bool isAvailable
Definition: ScheduleDAG.h:285
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
SUnit * getSUnit() const
Definition: ScheduleDAG.h:503
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
Sequence
A sequence of states that a pointer may go through in which an objc_retain and objc_release are actua...
Definition: PtrState.h:37
static bool CheckForLiveRegDef(SUnit *SU, unsigned Reg, std::vector< SUnit * > &LiveRegDefs, SmallSet< unsigned, 4 > &RegAdded, SmallVectorImpl< unsigned > &LRegs, const TargetRegisterInfo *TRI)
CheckForLiveRegDef - Return true and update live register vector if the specified register def of the...
static RegisterScheduler linearizeDAGScheduler("linearize","Linearize DAG, no scheduling", createDAGLinearizer)
int getNodeId() const
Return the unique node id.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:175
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned NodeNum
Definition: ScheduleDAG.h:266
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool addPred(const SDep &D, bool Required=true)
addPred - This adds the specified edge as a pred of the current node if not already.
Definition: ScheduleDAG.cpp:65
SmallVector< SDep, 4 > Succs
Definition: ScheduleDAG.h:259
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:210
Arbitrary strong DAG edge (no real dependence).
Definition: ScheduleDAG.h:68
ScheduleDAGSDNodes * createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level OptLevel)
createFastDAGScheduler - This creates a "fast" scheduler.
#define DEBUG(X)
Definition: Debug.h:100
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
void dump(const ScheduleDAG *G) const
SUnit - Scheduling unit.
SUnit - Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:244
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.