LLVM  3.7.0
ScheduleDAGInstrs.cpp
Go to the documentation of this file.
1 //===---- ScheduleDAGInstrs.cpp - MachineInstr Rescheduling ---------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements the ScheduleDAGInstrs class, which implements re-scheduling
11 // of MachineInstrs.
12 //
13 //===----------------------------------------------------------------------===//
14 
16 #include "llvm/ADT/MapVector.h"
17 #include "llvm/ADT/SmallPtrSet.h"
18 #include "llvm/ADT/SmallSet.h"
30 #include "llvm/IR/Operator.h"
32 #include "llvm/Support/Debug.h"
33 #include "llvm/Support/Format.h"
39 #include <queue>
40 
41 using namespace llvm;
42 
43 #define DEBUG_TYPE "misched"
44 
45 static cl::opt<bool> EnableAASchedMI("enable-aa-sched-mi", cl::Hidden,
46  cl::ZeroOrMore, cl::init(false),
47  cl::desc("Enable use of AA during MI DAG construction"));
48 
49 static cl::opt<bool> UseTBAA("use-tbaa-in-sched-mi", cl::Hidden,
50  cl::init(true), cl::desc("Enable use of TBAA during MI DAG construction"));
51 
53  const MachineLoopInfo *mli,
54  bool IsPostRAFlag, bool RemoveKillFlags,
55  LiveIntervals *lis)
56  : ScheduleDAG(mf), MLI(mli), MFI(mf.getFrameInfo()), LIS(lis),
57  IsPostRA(IsPostRAFlag), RemoveKillFlags(RemoveKillFlags),
58  CanHandleTerminators(false), FirstDbgValue(nullptr) {
59  assert((IsPostRA || LIS) && "PreRA scheduling requires LiveIntervals");
60  DbgValues.clear();
61  assert(!(IsPostRA && MRI.getNumVirtRegs()) &&
62  "Virtual registers must be removed prior to PostRA scheduling");
63 
64  const TargetSubtargetInfo &ST = mf.getSubtarget();
66 }
67 
68 /// getUnderlyingObjectFromInt - This is the function that does the work of
69 /// looking through basic ptrtoint+arithmetic+inttoptr sequences.
70 static const Value *getUnderlyingObjectFromInt(const Value *V) {
71  do {
72  if (const Operator *U = dyn_cast<Operator>(V)) {
73  // If we find a ptrtoint, we can transfer control back to the
74  // regular getUnderlyingObjectFromInt.
75  if (U->getOpcode() == Instruction::PtrToInt)
76  return U->getOperand(0);
77  // If we find an add of a constant, a multiplied value, or a phi, it's
78  // likely that the other operand will lead us to the base
79  // object. We don't have to worry about the case where the
80  // object address is somehow being computed by the multiply,
81  // because our callers only care when the result is an
82  // identifiable object.
83  if (U->getOpcode() != Instruction::Add ||
84  (!isa<ConstantInt>(U->getOperand(1)) &&
85  Operator::getOpcode(U->getOperand(1)) != Instruction::Mul &&
86  !isa<PHINode>(U->getOperand(1))))
87  return V;
88  V = U->getOperand(0);
89  } else {
90  return V;
91  }
92  assert(V->getType()->isIntegerTy() && "Unexpected operand type!");
93  } while (1);
94 }
95 
96 /// getUnderlyingObjects - This is a wrapper around GetUnderlyingObjects
97 /// and adds support for basic ptrtoint+arithmetic+inttoptr sequences.
98 static void getUnderlyingObjects(const Value *V,
99  SmallVectorImpl<Value *> &Objects,
100  const DataLayout &DL) {
102  SmallVector<const Value *, 4> Working(1, V);
103  do {
104  V = Working.pop_back_val();
105 
107  GetUnderlyingObjects(const_cast<Value *>(V), Objs, DL);
108 
109  for (SmallVectorImpl<Value *>::iterator I = Objs.begin(), IE = Objs.end();
110  I != IE; ++I) {
111  V = *I;
112  if (!Visited.insert(V).second)
113  continue;
114  if (Operator::getOpcode(V) == Instruction::IntToPtr) {
115  const Value *O =
116  getUnderlyingObjectFromInt(cast<User>(V)->getOperand(0));
117  if (O->getType()->isPointerTy()) {
118  Working.push_back(O);
119  continue;
120  }
121  }
122  Objects.push_back(const_cast<Value *>(V));
123  }
124  } while (!Working.empty());
125 }
126 
130 
131 /// getUnderlyingObjectsForInstr - If this machine instr has memory reference
132 /// information and it can be tracked to a normal reference to a known
133 /// object, return the Value for that object.
135  const MachineFrameInfo *MFI,
136  UnderlyingObjectsVector &Objects,
137  const DataLayout &DL) {
138  if (!MI->hasOneMemOperand() ||
139  (!(*MI->memoperands_begin())->getValue() &&
140  !(*MI->memoperands_begin())->getPseudoValue()) ||
141  (*MI->memoperands_begin())->isVolatile())
142  return;
143 
144  if (const PseudoSourceValue *PSV =
145  (*MI->memoperands_begin())->getPseudoValue()) {
146  // Function that contain tail calls don't have unique PseudoSourceValue
147  // objects. Two PseudoSourceValues might refer to the same or overlapping
148  // locations. The client code calling this function assumes this is not the
149  // case. So return a conservative answer of no known object.
150  if (MFI->hasTailCall())
151  return;
152 
153  // For now, ignore PseudoSourceValues which may alias LLVM IR values
154  // because the code that uses this function has no way to cope with
155  // such aliases.
156  if (!PSV->isAliased(MFI)) {
157  bool MayAlias = PSV->mayAlias(MFI);
158  Objects.push_back(UnderlyingObjectsVector::value_type(PSV, MayAlias));
159  }
160  return;
161  }
162 
163  const Value *V = (*MI->memoperands_begin())->getValue();
164  if (!V)
165  return;
166 
168  getUnderlyingObjects(V, Objs, DL);
169 
170  for (Value *V : Objs) {
171  if (!isIdentifiedObject(V)) {
172  Objects.clear();
173  return;
174  }
175 
177  }
178 }
179 
181  BB = bb;
182 }
183 
185  // Subclasses should no longer refer to the old block.
186  BB = nullptr;
187 }
188 
189 /// Initialize the DAG and common scheduler state for the current scheduling
190 /// region. This does not actually create the DAG, only clears it. The
191 /// scheduling driver may call BuildSchedGraph multiple times per scheduling
192 /// region.
196  unsigned regioninstrs) {
197  assert(bb == BB && "startBlock should set BB");
198  RegionBegin = begin;
199  RegionEnd = end;
200  NumRegionInstrs = regioninstrs;
201 }
202 
203 /// Close the current scheduling region. Don't clear any state in case the
204 /// driver wants to refer to the previous scheduling region.
206  // Nothing to do.
207 }
208 
209 /// addSchedBarrierDeps - Add dependencies from instructions in the current
210 /// list of instructions being scheduled to scheduling barrier by adding
211 /// the exit SU to the register defs and use list. This is because we want to
212 /// make sure instructions which define registers that are either used by
213 /// the terminator or are live-out are properly scheduled. This is
214 /// especially important when the definition latency of the return value(s)
215 /// are too high to be hidden by the branch or when the liveout registers
216 /// used by instructions in the fallthrough block.
218  MachineInstr *ExitMI = RegionEnd != BB->end() ? &*RegionEnd : nullptr;
219  ExitSU.setInstr(ExitMI);
220  bool AllDepKnown = ExitMI &&
221  (ExitMI->isCall() || ExitMI->isBarrier());
222  if (ExitMI && AllDepKnown) {
223  // If it's a call or a barrier, add dependencies on the defs and uses of
224  // instruction.
225  for (unsigned i = 0, e = ExitMI->getNumOperands(); i != e; ++i) {
226  const MachineOperand &MO = ExitMI->getOperand(i);
227  if (!MO.isReg() || MO.isDef()) continue;
228  unsigned Reg = MO.getReg();
229  if (Reg == 0) continue;
230 
231  if (TRI->isPhysicalRegister(Reg))
232  Uses.insert(PhysRegSUOper(&ExitSU, -1, Reg));
233  else {
234  assert(!IsPostRA && "Virtual register encountered after regalloc.");
235  if (MO.readsReg()) // ignore undef operands
236  addVRegUseDeps(&ExitSU, i);
237  }
238  }
239  } else {
240  // For others, e.g. fallthrough, conditional branch, assume the exit
241  // uses all the registers that are livein to the successor blocks.
242  assert(Uses.empty() && "Uses in set before adding deps?");
244  SE = BB->succ_end(); SI != SE; ++SI)
245  for (MachineBasicBlock::livein_iterator I = (*SI)->livein_begin(),
246  E = (*SI)->livein_end(); I != E; ++I) {
247  unsigned Reg = *I;
248  if (!Uses.contains(Reg))
249  Uses.insert(PhysRegSUOper(&ExitSU, -1, Reg));
250  }
251  }
252 }
253 
254 /// MO is an operand of SU's instruction that defines a physical register. Add
255 /// data dependencies from SU to any uses of the physical register.
256 void ScheduleDAGInstrs::addPhysRegDataDeps(SUnit *SU, unsigned OperIdx) {
257  const MachineOperand &MO = SU->getInstr()->getOperand(OperIdx);
258  assert(MO.isDef() && "expect physreg def");
259 
260  // Ask the target if address-backscheduling is desirable, and if so how much.
262 
263  for (MCRegAliasIterator Alias(MO.getReg(), TRI, true);
264  Alias.isValid(); ++Alias) {
265  if (!Uses.contains(*Alias))
266  continue;
267  for (Reg2SUnitsMap::iterator I = Uses.find(*Alias); I != Uses.end(); ++I) {
268  SUnit *UseSU = I->SU;
269  if (UseSU == SU)
270  continue;
271 
272  // Adjust the dependence latency using operand def/use information,
273  // then allow the target to perform its own adjustments.
274  int UseOp = I->OpIdx;
275  MachineInstr *RegUse = nullptr;
276  SDep Dep;
277  if (UseOp < 0)
278  Dep = SDep(SU, SDep::Artificial);
279  else {
280  // Set the hasPhysRegDefs only for physreg defs that have a use within
281  // the scheduling region.
282  SU->hasPhysRegDefs = true;
283  Dep = SDep(SU, SDep::Data, *Alias);
284  RegUse = UseSU->getInstr();
285  }
286  Dep.setLatency(
287  SchedModel.computeOperandLatency(SU->getInstr(), OperIdx, RegUse,
288  UseOp));
289 
290  ST.adjustSchedDependency(SU, UseSU, Dep);
291  UseSU->addPred(Dep);
292  }
293  }
294 }
295 
296 /// addPhysRegDeps - Add register dependencies (data, anti, and output) from
297 /// this SUnit to following instructions in the same scheduling region that
298 /// depend the physical register referenced at OperIdx.
299 void ScheduleDAGInstrs::addPhysRegDeps(SUnit *SU, unsigned OperIdx) {
300  MachineInstr *MI = SU->getInstr();
301  MachineOperand &MO = MI->getOperand(OperIdx);
302 
303  // Optionally add output and anti dependencies. For anti
304  // dependencies we use a latency of 0 because for a multi-issue
305  // target we want to allow the defining instruction to issue
306  // in the same cycle as the using instruction.
307  // TODO: Using a latency of 1 here for output dependencies assumes
308  // there's no cost for reusing registers.
310  for (MCRegAliasIterator Alias(MO.getReg(), TRI, true);
311  Alias.isValid(); ++Alias) {
312  if (!Defs.contains(*Alias))
313  continue;
314  for (Reg2SUnitsMap::iterator I = Defs.find(*Alias); I != Defs.end(); ++I) {
315  SUnit *DefSU = I->SU;
316  if (DefSU == &ExitSU)
317  continue;
318  if (DefSU != SU &&
319  (Kind != SDep::Output || !MO.isDead() ||
320  !DefSU->getInstr()->registerDefIsDead(*Alias))) {
321  if (Kind == SDep::Anti)
322  DefSU->addPred(SDep(SU, Kind, /*Reg=*/*Alias));
323  else {
324  SDep Dep(SU, Kind, /*Reg=*/*Alias);
325  Dep.setLatency(
326  SchedModel.computeOutputLatency(MI, OperIdx, DefSU->getInstr()));
327  DefSU->addPred(Dep);
328  }
329  }
330  }
331  }
332 
333  if (!MO.isDef()) {
334  SU->hasPhysRegUses = true;
335  // Either insert a new Reg2SUnits entry with an empty SUnits list, or
336  // retrieve the existing SUnits list for this register's uses.
337  // Push this SUnit on the use list.
338  Uses.insert(PhysRegSUOper(SU, OperIdx, MO.getReg()));
339  if (RemoveKillFlags)
340  MO.setIsKill(false);
341  }
342  else {
343  addPhysRegDataDeps(SU, OperIdx);
344  unsigned Reg = MO.getReg();
345 
346  // clear this register's use list
347  if (Uses.contains(Reg))
348  Uses.eraseAll(Reg);
349 
350  if (!MO.isDead()) {
351  Defs.eraseAll(Reg);
352  } else if (SU->isCall) {
353  // Calls will not be reordered because of chain dependencies (see
354  // below). Since call operands are dead, calls may continue to be added
355  // to the DefList making dependence checking quadratic in the size of
356  // the block. Instead, we leave only one call at the back of the
357  // DefList.
359  Reg2SUnitsMap::iterator B = P.first;
360  Reg2SUnitsMap::iterator I = P.second;
361  for (bool isBegin = I == B; !isBegin; /* empty */) {
362  isBegin = (--I) == B;
363  if (!I->SU->isCall)
364  break;
365  I = Defs.erase(I);
366  }
367  }
368 
369  // Defs are pushed in the order they are visited and never reordered.
370  Defs.insert(PhysRegSUOper(SU, OperIdx, Reg));
371  }
372 }
373 
374 /// addVRegDefDeps - Add register output and data dependencies from this SUnit
375 /// to instructions that occur later in the same scheduling region if they read
376 /// from or write to the virtual register defined at OperIdx.
377 ///
378 /// TODO: Hoist loop induction variable increments. This has to be
379 /// reevaluated. Generally, IV scheduling should be done before coalescing.
380 void ScheduleDAGInstrs::addVRegDefDeps(SUnit *SU, unsigned OperIdx) {
381  const MachineInstr *MI = SU->getInstr();
382  unsigned Reg = MI->getOperand(OperIdx).getReg();
383 
384  // Singly defined vregs do not have output/anti dependencies.
385  // The current operand is a def, so we have at least one.
386  // Check here if there are any others...
387  if (MRI.hasOneDef(Reg))
388  return;
389 
390  // Add output dependence to the next nearest def of this vreg.
391  //
392  // Unless this definition is dead, the output dependence should be
393  // transitively redundant with antidependencies from this definition's
394  // uses. We're conservative for now until we have a way to guarantee the uses
395  // are not eliminated sometime during scheduling. The output dependence edge
396  // is also useful if output latency exceeds def-use latency.
398  if (DefI == VRegDefs.end())
399  VRegDefs.insert(VReg2SUnit(Reg, SU));
400  else {
401  SUnit *DefSU = DefI->SU;
402  if (DefSU != SU && DefSU != &ExitSU) {
403  SDep Dep(SU, SDep::Output, Reg);
404  Dep.setLatency(
405  SchedModel.computeOutputLatency(MI, OperIdx, DefSU->getInstr()));
406  DefSU->addPred(Dep);
407  }
408  DefI->SU = SU;
409  }
410 }
411 
412 /// addVRegUseDeps - Add a register data dependency if the instruction that
413 /// defines the virtual register used at OperIdx is mapped to an SUnit. Add a
414 /// register antidependency from this SUnit to instructions that occur later in
415 /// the same scheduling region if they write the virtual register.
416 ///
417 /// TODO: Handle ExitSU "uses" properly.
418 void ScheduleDAGInstrs::addVRegUseDeps(SUnit *SU, unsigned OperIdx) {
419  MachineInstr *MI = SU->getInstr();
420  unsigned Reg = MI->getOperand(OperIdx).getReg();
421 
422  // Record this local VReg use.
424  for (; UI != VRegUses.end(); ++UI) {
425  if (UI->SU == SU)
426  break;
427  }
428  if (UI == VRegUses.end())
429  VRegUses.insert(VReg2SUnit(Reg, SU));
430 
431  // Lookup this operand's reaching definition.
432  assert(LIS && "vreg dependencies requires LiveIntervals");
433  LiveQueryResult LRQ
435  VNInfo *VNI = LRQ.valueIn();
436 
437  // VNI will be valid because MachineOperand::readsReg() is checked by caller.
438  assert(VNI && "No value to read by operand");
440  // Phis and other noninstructions (after coalescing) have a NULL Def.
441  if (Def) {
442  SUnit *DefSU = getSUnit(Def);
443  if (DefSU) {
444  // The reaching Def lives within this scheduling region.
445  // Create a data dependence.
446  SDep dep(DefSU, SDep::Data, Reg);
447  // Adjust the dependence latency using operand def/use information, then
448  // allow the target to perform its own adjustments.
449  int DefOp = Def->findRegisterDefOperandIdx(Reg);
450  dep.setLatency(SchedModel.computeOperandLatency(Def, DefOp, MI, OperIdx));
451 
453  ST.adjustSchedDependency(DefSU, SU, const_cast<SDep &>(dep));
454  SU->addPred(dep);
455  }
456  }
457 
458  // Add antidependence to the following def of the vreg it uses.
460  if (DefI != VRegDefs.end() && DefI->SU != SU)
461  DefI->SU->addPred(SDep(SU, SDep::Anti, Reg));
462 }
463 
464 /// Return true if MI is an instruction we are unable to reason about
465 /// (like a call or something with unmodeled side effects).
467  if (MI->isCall() || MI->hasUnmodeledSideEffects() ||
468  (MI->hasOrderedMemoryRef() &&
469  (!MI->mayLoad() || !MI->isInvariantLoad(AA))))
470  return true;
471  return false;
472 }
473 
474 // This MI might have either incomplete info, or known to be unsafe
475 // to deal with (i.e. volatile object).
477  const MachineFrameInfo *MFI,
478  const DataLayout &DL) {
479  if (!MI || MI->memoperands_empty())
480  return true;
481  // We purposefully do no check for hasOneMemOperand() here
482  // in hope to trigger an assert downstream in order to
483  // finish implementation.
484  if ((*MI->memoperands_begin())->isVolatile() ||
486  return true;
487 
488  if ((*MI->memoperands_begin())->getPseudoValue()) {
489  // Similarly to getUnderlyingObjectForInstr:
490  // For now, ignore PseudoSourceValues which may alias LLVM IR values
491  // because the code that uses this function has no way to cope with
492  // such aliases.
493  return true;
494  }
495 
496  const Value *V = (*MI->memoperands_begin())->getValue();
497  if (!V)
498  return true;
499 
501  getUnderlyingObjects(V, Objs, DL);
502  for (Value *V : Objs) {
503  // Does this pointer refer to a distinct and identifiable object?
504  if (!isIdentifiedObject(V))
505  return true;
506  }
507 
508  return false;
509 }
510 
511 /// This returns true if the two MIs need a chain edge betwee them.
512 /// If these are not even memory operations, we still may need
513 /// chain deps between them. The question really is - could
514 /// these two MIs be reordered during scheduling from memory dependency
515 /// point of view.
516 static bool MIsNeedChainEdge(AliasAnalysis *AA, const MachineFrameInfo *MFI,
517  const DataLayout &DL, MachineInstr *MIa,
518  MachineInstr *MIb) {
519  const MachineFunction *MF = MIa->getParent()->getParent();
520  const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
521 
522  // Cover a trivial case - no edge is need to itself.
523  if (MIa == MIb)
524  return false;
525 
526  // Let the target decide if memory accesses cannot possibly overlap.
527  if ((MIa->mayLoad() || MIa->mayStore()) &&
528  (MIb->mayLoad() || MIb->mayStore()))
529  if (TII->areMemAccessesTriviallyDisjoint(MIa, MIb, AA))
530  return false;
531 
532  // FIXME: Need to handle multiple memory operands to support all targets.
533  if (!MIa->hasOneMemOperand() || !MIb->hasOneMemOperand())
534  return true;
535 
536  if (isUnsafeMemoryObject(MIa, MFI, DL) || isUnsafeMemoryObject(MIb, MFI, DL))
537  return true;
538 
539  // If we are dealing with two "normal" loads, we do not need an edge
540  // between them - they could be reordered.
541  if (!MIa->mayStore() && !MIb->mayStore())
542  return false;
543 
544  // To this point analysis is generic. From here on we do need AA.
545  if (!AA)
546  return true;
547 
548  MachineMemOperand *MMOa = *MIa->memoperands_begin();
549  MachineMemOperand *MMOb = *MIb->memoperands_begin();
550 
551  if (!MMOa->getValue() || !MMOb->getValue())
552  return true;
553 
554  // The following interface to AA is fashioned after DAGCombiner::isAlias
555  // and operates with MachineMemOperand offset with some important
556  // assumptions:
557  // - LLVM fundamentally assumes flat address spaces.
558  // - MachineOperand offset can *only* result from legalization and
559  // cannot affect queries other than the trivial case of overlap
560  // checking.
561  // - These offsets never wrap and never step outside
562  // of allocated objects.
563  // - There should never be any negative offsets here.
564  //
565  // FIXME: Modify API to hide this math from "user"
566  // FIXME: Even before we go to AA we can reason locally about some
567  // memory objects. It can save compile time, and possibly catch some
568  // corner cases not currently covered.
569 
570  assert ((MMOa->getOffset() >= 0) && "Negative MachineMemOperand offset");
571  assert ((MMOb->getOffset() >= 0) && "Negative MachineMemOperand offset");
572 
573  int64_t MinOffset = std::min(MMOa->getOffset(), MMOb->getOffset());
574  int64_t Overlapa = MMOa->getSize() + MMOa->getOffset() - MinOffset;
575  int64_t Overlapb = MMOb->getSize() + MMOb->getOffset() - MinOffset;
576 
577  AliasResult AAResult =
578  AA->alias(MemoryLocation(MMOa->getValue(), Overlapa,
579  UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
580  MemoryLocation(MMOb->getValue(), Overlapb,
581  UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
582 
583  return (AAResult != NoAlias);
584 }
585 
586 /// This recursive function iterates over chain deps of SUb looking for
587 /// "latest" node that needs a chain edge to SUa.
588 static unsigned iterateChainSucc(AliasAnalysis *AA, const MachineFrameInfo *MFI,
589  const DataLayout &DL, SUnit *SUa, SUnit *SUb,
590  SUnit *ExitSU, unsigned *Depth,
592  if (!SUa || !SUb || SUb == ExitSU)
593  return *Depth;
594 
595  // Remember visited nodes.
596  if (!Visited.insert(SUb).second)
597  return *Depth;
598  // If there is _some_ dependency already in place, do not
599  // descend any further.
600  // TODO: Need to make sure that if that dependency got eliminated or ignored
601  // for any reason in the future, we would not violate DAG topology.
602  // Currently it does not happen, but makes an implicit assumption about
603  // future implementation.
604  //
605  // Independently, if we encounter node that is some sort of global
606  // object (like a call) we already have full set of dependencies to it
607  // and we can stop descending.
608  if (SUa->isSucc(SUb) ||
609  isGlobalMemoryObject(AA, SUb->getInstr()))
610  return *Depth;
611 
612  // If we do need an edge, or we have exceeded depth budget,
613  // add that edge to the predecessors chain of SUb,
614  // and stop descending.
615  if (*Depth > 200 ||
616  MIsNeedChainEdge(AA, MFI, DL, SUa->getInstr(), SUb->getInstr())) {
617  SUb->addPred(SDep(SUa, SDep::MayAliasMem));
618  return *Depth;
619  }
620  // Track current depth.
621  (*Depth)++;
622  // Iterate over memory dependencies only.
623  for (SUnit::const_succ_iterator I = SUb->Succs.begin(), E = SUb->Succs.end();
624  I != E; ++I)
625  if (I->isNormalMemoryOrBarrier())
626  iterateChainSucc(AA, MFI, DL, SUa, I->getSUnit(), ExitSU, Depth, Visited);
627  return *Depth;
628 }
629 
630 /// This function assumes that "downward" from SU there exist
631 /// tail/leaf of already constructed DAG. It iterates downward and
632 /// checks whether SU can be aliasing any node dominated
633 /// by it.
634 static void adjustChainDeps(AliasAnalysis *AA, const MachineFrameInfo *MFI,
635  const DataLayout &DL, SUnit *SU, SUnit *ExitSU,
636  std::set<SUnit *> &CheckList,
637  unsigned LatencyToLoad) {
638  if (!SU)
639  return;
640 
642  unsigned Depth = 0;
643 
644  for (std::set<SUnit *>::iterator I = CheckList.begin(), IE = CheckList.end();
645  I != IE; ++I) {
646  if (SU == *I)
647  continue;
648  if (MIsNeedChainEdge(AA, MFI, DL, SU->getInstr(), (*I)->getInstr())) {
649  SDep Dep(SU, SDep::MayAliasMem);
650  Dep.setLatency(((*I)->getInstr()->mayLoad()) ? LatencyToLoad : 0);
651  (*I)->addPred(Dep);
652  }
653 
654  // Iterate recursively over all previously added memory chain
655  // successors. Keep track of visited nodes.
656  for (SUnit::const_succ_iterator J = (*I)->Succs.begin(),
657  JE = (*I)->Succs.end(); J != JE; ++J)
658  if (J->isNormalMemoryOrBarrier())
659  iterateChainSucc(AA, MFI, DL, SU, J->getSUnit(), ExitSU, &Depth,
660  Visited);
661  }
662 }
663 
664 /// Check whether two objects need a chain edge, if so, add it
665 /// otherwise remember the rejected SU.
666 static inline void addChainDependency(AliasAnalysis *AA,
667  const MachineFrameInfo *MFI,
668  const DataLayout &DL, SUnit *SUa,
669  SUnit *SUb, std::set<SUnit *> &RejectList,
670  unsigned TrueMemOrderLatency = 0,
671  bool isNormalMemory = false) {
672  // If this is a false dependency,
673  // do not add the edge, but rememeber the rejected node.
674  if (MIsNeedChainEdge(AA, MFI, DL, SUa->getInstr(), SUb->getInstr())) {
675  SDep Dep(SUa, isNormalMemory ? SDep::MayAliasMem : SDep::Barrier);
676  Dep.setLatency(TrueMemOrderLatency);
677  SUb->addPred(Dep);
678  }
679  else {
680  // Duplicate entries should be ignored.
681  RejectList.insert(SUb);
682  DEBUG(dbgs() << "\tReject chain dep between SU("
683  << SUa->NodeNum << ") and SU("
684  << SUb->NodeNum << ")\n");
685  }
686 }
687 
688 /// Create an SUnit for each real instruction, numbered in top-down toplological
689 /// order. The instruction order A < B, implies that no edge exists from B to A.
690 ///
691 /// Map each real instruction to its SUnit.
692 ///
693 /// After initSUnits, the SUnits vector cannot be resized and the scheduler may
694 /// hang onto SUnit pointers. We may relax this in the future by using SUnit IDs
695 /// instead of pointers.
696 ///
697 /// MachineScheduler relies on initSUnits numbering the nodes by their order in
698 /// the original instruction list.
700  // We'll be allocating one SUnit for each real instruction in the region,
701  // which is contained within a basic block.
702  SUnits.reserve(NumRegionInstrs);
703 
705  MachineInstr *MI = I;
706  if (MI->isDebugValue())
707  continue;
708 
709  SUnit *SU = newSUnit(MI);
710  MISUnitMap[MI] = SU;
711 
712  SU->isCall = MI->isCall();
713  SU->isCommutable = MI->isCommutable();
714 
715  // Assign the Latency field of SU using target-provided information.
716  SU->Latency = SchedModel.computeInstrLatency(SU->getInstr());
717 
718  // If this SUnit uses a reserved or unbuffered resource, mark it as such.
719  //
720  // Reserved resources block an instruction from issuing and stall the
721  // entire pipeline. These are identified by BufferSize=0.
722  //
723  // Unbuffered resources prevent execution of subsequent instructions that
724  // require the same resources. This is used for in-order execution pipelines
725  // within an out-of-order core. These are identified by BufferSize=1.
727  const MCSchedClassDesc *SC = getSchedClass(SU);
730  PE = SchedModel.getWriteProcResEnd(SC); PI != PE; ++PI) {
731  switch (SchedModel.getProcResource(PI->ProcResourceIdx)->BufferSize) {
732  case 0:
733  SU->hasReservedResource = true;
734  break;
735  case 1:
736  SU->isUnbuffered = true;
737  break;
738  default:
739  break;
740  }
741  }
742  }
743  }
744 }
745 
746 /// If RegPressure is non-null, compute register pressure as a side effect. The
747 /// DAG builder is an efficient place to do it because it already visits
748 /// operands.
750  RegPressureTracker *RPTracker,
751  PressureDiffs *PDiffs) {
753  bool UseAA = EnableAASchedMI.getNumOccurrences() > 0 ? EnableAASchedMI
754  : ST.useAA();
755  AliasAnalysis *AAForDep = UseAA ? AA : nullptr;
756 
757  MISUnitMap.clear();
759 
760  // Create an SUnit for each real instruction.
761  initSUnits();
762 
763  if (PDiffs)
764  PDiffs->init(SUnits.size());
765 
766  // We build scheduling units by walking a block's instruction list from bottom
767  // to top.
768 
769  // Remember where a generic side-effecting instruction is as we procede.
770  SUnit *BarrierChain = nullptr, *AliasChain = nullptr;
771 
772  // Memory references to specific known memory locations are tracked
773  // so that they can be given more precise dependencies. We track
774  // separately the known memory locations that may alias and those
775  // that are known not to alias
776  MapVector<ValueType, std::vector<SUnit *> > AliasMemDefs, NonAliasMemDefs;
777  MapVector<ValueType, std::vector<SUnit *> > AliasMemUses, NonAliasMemUses;
778  std::set<SUnit*> RejectMemNodes;
779 
780  // Remove any stale debug info; sometimes BuildSchedGraph is called again
781  // without emitting the info from the previous call.
782  DbgValues.clear();
783  FirstDbgValue = nullptr;
784 
785  assert(Defs.empty() && Uses.empty() &&
786  "Only BuildGraph should update Defs/Uses");
789 
790  assert(VRegDefs.empty() && "Only BuildSchedGraph may access VRegDefs");
791  VRegUses.clear();
794 
795  // Model data dependencies between instructions being scheduled and the
796  // ExitSU.
798 
799  // Walk the list of instructions, from bottom moving up.
800  MachineInstr *DbgMI = nullptr;
802  MII != MIE; --MII) {
803  MachineInstr *MI = std::prev(MII);
804  if (MI && DbgMI) {
805  DbgValues.push_back(std::make_pair(DbgMI, MI));
806  DbgMI = nullptr;
807  }
808 
809  if (MI->isDebugValue()) {
810  DbgMI = MI;
811  continue;
812  }
813  SUnit *SU = MISUnitMap[MI];
814  assert(SU && "No SUnit mapped to this MI");
815 
816  if (RPTracker) {
817  PressureDiff *PDiff = PDiffs ? &(*PDiffs)[SU->NodeNum] : nullptr;
818  RPTracker->recede(/*LiveUses=*/nullptr, PDiff);
819  assert(RPTracker->getPos() == std::prev(MII) &&
820  "RPTracker can't find MI");
821  }
822 
823  assert(
824  (CanHandleTerminators || (!MI->isTerminator() && !MI->isPosition())) &&
825  "Cannot schedule terminators or labels!");
826 
827  // Add register-based dependencies (data, anti, and output).
828  bool HasVRegDef = false;
829  for (unsigned j = 0, n = MI->getNumOperands(); j != n; ++j) {
830  const MachineOperand &MO = MI->getOperand(j);
831  if (!MO.isReg()) continue;
832  unsigned Reg = MO.getReg();
833  if (Reg == 0) continue;
834 
835  if (TRI->isPhysicalRegister(Reg))
836  addPhysRegDeps(SU, j);
837  else {
838  assert(!IsPostRA && "Virtual register encountered!");
839  if (MO.isDef()) {
840  HasVRegDef = true;
841  addVRegDefDeps(SU, j);
842  }
843  else if (MO.readsReg()) // ignore undef operands
844  addVRegUseDeps(SU, j);
845  }
846  }
847  // If we haven't seen any uses in this scheduling region, create a
848  // dependence edge to ExitSU to model the live-out latency. This is required
849  // for vreg defs with no in-region use, and prefetches with no vreg def.
850  //
851  // FIXME: NumDataSuccs would be more precise than NumSuccs here. This
852  // check currently relies on being called before adding chain deps.
853  if (SU->NumSuccs == 0 && SU->Latency > 1
854  && (HasVRegDef || MI->mayLoad())) {
855  SDep Dep(SU, SDep::Artificial);
856  Dep.setLatency(SU->Latency - 1);
857  ExitSU.addPred(Dep);
858  }
859 
860  // Add chain dependencies.
861  // Chain dependencies used to enforce memory order should have
862  // latency of 0 (except for true dependency of Store followed by
863  // aliased Load... we estimate that with a single cycle of latency
864  // assuming the hardware will bypass)
865  // Note that isStoreToStackSlot and isLoadFromStackSLot are not usable
866  // after stack slots are lowered to actual addresses.
867  // TODO: Use an AliasAnalysis and do real alias-analysis queries, and
868  // produce more precise dependence information.
869  unsigned TrueMemOrderLatency = MI->mayStore() ? 1 : 0;
870  if (isGlobalMemoryObject(AA, MI)) {
871  // Be conservative with these and add dependencies on all memory
872  // references, even those that are known to not alias.
873  for (MapVector<ValueType, std::vector<SUnit *> >::iterator I =
874  NonAliasMemDefs.begin(), E = NonAliasMemDefs.end(); I != E; ++I) {
875  for (unsigned i = 0, e = I->second.size(); i != e; ++i) {
876  I->second[i]->addPred(SDep(SU, SDep::Barrier));
877  }
878  }
879  for (MapVector<ValueType, std::vector<SUnit *> >::iterator I =
880  NonAliasMemUses.begin(), E = NonAliasMemUses.end(); I != E; ++I) {
881  for (unsigned i = 0, e = I->second.size(); i != e; ++i) {
882  SDep Dep(SU, SDep::Barrier);
883  Dep.setLatency(TrueMemOrderLatency);
884  I->second[i]->addPred(Dep);
885  }
886  }
887  // Add SU to the barrier chain.
888  if (BarrierChain)
889  BarrierChain->addPred(SDep(SU, SDep::Barrier));
890  BarrierChain = SU;
891  // This is a barrier event that acts as a pivotal node in the DAG,
892  // so it is safe to clear list of exposed nodes.
893  adjustChainDeps(AA, MFI, *TM.getDataLayout(), SU, &ExitSU, RejectMemNodes,
894  TrueMemOrderLatency);
895  RejectMemNodes.clear();
896  NonAliasMemDefs.clear();
897  NonAliasMemUses.clear();
898 
899  // fall-through
900  new_alias_chain:
901  // Chain all possibly aliasing memory references through SU.
902  if (AliasChain) {
903  unsigned ChainLatency = 0;
904  if (AliasChain->getInstr()->mayLoad())
905  ChainLatency = TrueMemOrderLatency;
906  addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, AliasChain,
907  RejectMemNodes, ChainLatency);
908  }
909  AliasChain = SU;
910  for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k)
911  addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
912  PendingLoads[k], RejectMemNodes,
913  TrueMemOrderLatency);
914  for (MapVector<ValueType, std::vector<SUnit *> >::iterator I =
915  AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I) {
916  for (unsigned i = 0, e = I->second.size(); i != e; ++i)
917  addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
918  I->second[i], RejectMemNodes);
919  }
920  for (MapVector<ValueType, std::vector<SUnit *> >::iterator I =
921  AliasMemUses.begin(), E = AliasMemUses.end(); I != E; ++I) {
922  for (unsigned i = 0, e = I->second.size(); i != e; ++i)
923  addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
924  I->second[i], RejectMemNodes, TrueMemOrderLatency);
925  }
926  adjustChainDeps(AA, MFI, *TM.getDataLayout(), SU, &ExitSU, RejectMemNodes,
927  TrueMemOrderLatency);
928  PendingLoads.clear();
929  AliasMemDefs.clear();
930  AliasMemUses.clear();
931  } else if (MI->mayStore()) {
932  // Add dependence on barrier chain, if needed.
933  // There is no point to check aliasing on barrier event. Even if
934  // SU and barrier _could_ be reordered, they should not. In addition,
935  // we have lost all RejectMemNodes below barrier.
936  if (BarrierChain)
937  BarrierChain->addPred(SDep(SU, SDep::Barrier));
938 
941 
942  if (Objs.empty()) {
943  // Treat all other stores conservatively.
944  goto new_alias_chain;
945  }
946 
947  bool MayAlias = false;
948  for (UnderlyingObjectsVector::iterator K = Objs.begin(), KE = Objs.end();
949  K != KE; ++K) {
950  ValueType V = K->getPointer();
951  bool ThisMayAlias = K->getInt();
952  if (ThisMayAlias)
953  MayAlias = true;
954 
955  // A store to a specific PseudoSourceValue. Add precise dependencies.
956  // Record the def in MemDefs, first adding a dep if there is
957  // an existing def.
959  ((ThisMayAlias) ? AliasMemDefs.find(V) : NonAliasMemDefs.find(V));
961  ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end());
962  if (I != IE) {
963  for (unsigned i = 0, e = I->second.size(); i != e; ++i)
964  addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
965  I->second[i], RejectMemNodes, 0, true);
966 
967  // If we're not using AA, then we only need one store per object.
968  if (!AAForDep)
969  I->second.clear();
970  I->second.push_back(SU);
971  } else {
972  if (ThisMayAlias) {
973  if (!AAForDep)
974  AliasMemDefs[V].clear();
975  AliasMemDefs[V].push_back(SU);
976  } else {
977  if (!AAForDep)
978  NonAliasMemDefs[V].clear();
979  NonAliasMemDefs[V].push_back(SU);
980  }
981  }
982  // Handle the uses in MemUses, if there are any.
984  ((ThisMayAlias) ? AliasMemUses.find(V) : NonAliasMemUses.find(V));
986  ((ThisMayAlias) ? AliasMemUses.end() : NonAliasMemUses.end());
987  if (J != JE) {
988  for (unsigned i = 0, e = J->second.size(); i != e; ++i)
989  addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
990  J->second[i], RejectMemNodes,
991  TrueMemOrderLatency, true);
992  J->second.clear();
993  }
994  }
995  if (MayAlias) {
996  // Add dependencies from all the PendingLoads, i.e. loads
997  // with no underlying object.
998  for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k)
999  addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
1000  PendingLoads[k], RejectMemNodes,
1001  TrueMemOrderLatency);
1002  // Add dependence on alias chain, if needed.
1003  if (AliasChain)
1004  addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, AliasChain,
1005  RejectMemNodes);
1006  }
1007  adjustChainDeps(AA, MFI, *TM.getDataLayout(), SU, &ExitSU, RejectMemNodes,
1008  TrueMemOrderLatency);
1009  } else if (MI->mayLoad()) {
1010  bool MayAlias = true;
1011  if (MI->isInvariantLoad(AA)) {
1012  // Invariant load, no chain dependencies needed!
1013  } else {
1016 
1017  if (Objs.empty()) {
1018  // A load with no underlying object. Depend on all
1019  // potentially aliasing stores.
1020  for (MapVector<ValueType, std::vector<SUnit *> >::iterator I =
1021  AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I)
1022  for (unsigned i = 0, e = I->second.size(); i != e; ++i)
1023  addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
1024  I->second[i], RejectMemNodes);
1025 
1026  PendingLoads.push_back(SU);
1027  MayAlias = true;
1028  } else {
1029  MayAlias = false;
1030  }
1031 
1033  J = Objs.begin(), JE = Objs.end(); J != JE; ++J) {
1034  ValueType V = J->getPointer();
1035  bool ThisMayAlias = J->getInt();
1036 
1037  if (ThisMayAlias)
1038  MayAlias = true;
1039 
1040  // A load from a specific PseudoSourceValue. Add precise dependencies.
1042  ((ThisMayAlias) ? AliasMemDefs.find(V) : NonAliasMemDefs.find(V));
1044  ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end());
1045  if (I != IE)
1046  for (unsigned i = 0, e = I->second.size(); i != e; ++i)
1047  addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU,
1048  I->second[i], RejectMemNodes, 0, true);
1049  if (ThisMayAlias)
1050  AliasMemUses[V].push_back(SU);
1051  else
1052  NonAliasMemUses[V].push_back(SU);
1053  }
1054  if (MayAlias)
1055  adjustChainDeps(AA, MFI, *TM.getDataLayout(), SU, &ExitSU,
1056  RejectMemNodes, /*Latency=*/0);
1057  // Add dependencies on alias and barrier chains, if needed.
1058  if (MayAlias && AliasChain)
1059  addChainDependency(AAForDep, MFI, *TM.getDataLayout(), SU, AliasChain,
1060  RejectMemNodes);
1061  if (BarrierChain)
1062  BarrierChain->addPred(SDep(SU, SDep::Barrier));
1063  }
1064  }
1065  }
1066  if (DbgMI)
1067  FirstDbgValue = DbgMI;
1068 
1069  Defs.clear();
1070  Uses.clear();
1071  VRegDefs.clear();
1072  PendingLoads.clear();
1073 }
1074 
1075 /// \brief Initialize register live-range state for updating kills.
1077  // Start with no live registers.
1078  LiveRegs.reset();
1079 
1080  // Examine the live-in regs of all successors.
1082  SE = BB->succ_end(); SI != SE; ++SI) {
1083  for (MachineBasicBlock::livein_iterator I = (*SI)->livein_begin(),
1084  E = (*SI)->livein_end(); I != E; ++I) {
1085  unsigned Reg = *I;
1086  // Repeat, for reg and all subregs.
1087  for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
1088  SubRegs.isValid(); ++SubRegs)
1089  LiveRegs.set(*SubRegs);
1090  }
1091  }
1092 }
1093 
1094 /// \brief If we change a kill flag on the bundle instruction implicit register
1095 /// operands, then we also need to propagate that to any instructions inside
1096 /// the bundle which had the same kill state.
1097 static void toggleBundleKillFlag(MachineInstr *MI, unsigned Reg,
1098  bool NewKillState) {
1099  if (MI->getOpcode() != TargetOpcode::BUNDLE)
1100  return;
1101 
1102  // Walk backwards from the last instruction in the bundle to the first.
1103  // Once we set a kill flag on an instruction, we bail out, as otherwise we
1104  // might set it on too many operands. We will clear as many flags as we
1105  // can though.
1108  while (Begin != End) {
1109  for (MachineOperand &MO : (--End)->operands()) {
1110  if (!MO.isReg() || MO.isDef() || Reg != MO.getReg())
1111  continue;
1112 
1113  // DEBUG_VALUE nodes do not contribute to code generation and should
1114  // always be ignored. Failure to do so may result in trying to modify
1115  // KILL flags on DEBUG_VALUE nodes, which is distressing.
1116  if (MO.isDebug())
1117  continue;
1118 
1119  // If the register has the internal flag then it could be killing an
1120  // internal def of the register. In this case, just skip. We only want
1121  // to toggle the flag on operands visible outside the bundle.
1122  if (MO.isInternalRead())
1123  continue;
1124 
1125  if (MO.isKill() == NewKillState)
1126  continue;
1127  MO.setIsKill(NewKillState);
1128  if (NewKillState)
1129  return;
1130  }
1131  }
1132 }
1133 
1135  // Setting kill flag...
1136  if (!MO.isKill()) {
1137  MO.setIsKill(true);
1138  toggleBundleKillFlag(MI, MO.getReg(), true);
1139  return false;
1140  }
1141 
1142  // If MO itself is live, clear the kill flag...
1143  if (LiveRegs.test(MO.getReg())) {
1144  MO.setIsKill(false);
1145  toggleBundleKillFlag(MI, MO.getReg(), false);
1146  return false;
1147  }
1148 
1149  // If any subreg of MO is live, then create an imp-def for that
1150  // subreg and keep MO marked as killed.
1151  MO.setIsKill(false);
1152  toggleBundleKillFlag(MI, MO.getReg(), false);
1153  bool AllDead = true;
1154  const unsigned SuperReg = MO.getReg();
1155  MachineInstrBuilder MIB(MF, MI);
1156  for (MCSubRegIterator SubRegs(SuperReg, TRI); SubRegs.isValid(); ++SubRegs) {
1157  if (LiveRegs.test(*SubRegs)) {
1158  MIB.addReg(*SubRegs, RegState::ImplicitDefine);
1159  AllDead = false;
1160  }
1161  }
1162 
1163  if(AllDead) {
1164  MO.setIsKill(true);
1165  toggleBundleKillFlag(MI, MO.getReg(), true);
1166  }
1167  return false;
1168 }
1169 
1170 // FIXME: Reuse the LivePhysRegs utility for this.
1172  DEBUG(dbgs() << "Fixup kills for BB#" << MBB->getNumber() << '\n');
1173 
1175  BitVector killedRegs(TRI->getNumRegs());
1176 
1177  startBlockForKills(MBB);
1178 
1179  // Examine block from end to start...
1180  unsigned Count = MBB->size();
1181  for (MachineBasicBlock::iterator I = MBB->end(), E = MBB->begin();
1182  I != E; --Count) {
1183  MachineInstr *MI = --I;
1184  if (MI->isDebugValue())
1185  continue;
1186 
1187  // Update liveness. Registers that are defed but not used in this
1188  // instruction are now dead. Mark register and all subregs as they
1189  // are completely defined.
1190  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
1191  MachineOperand &MO = MI->getOperand(i);
1192  if (MO.isRegMask())
1194  if (!MO.isReg()) continue;
1195  unsigned Reg = MO.getReg();
1196  if (Reg == 0) continue;
1197  if (!MO.isDef()) continue;
1198  // Ignore two-addr defs.
1199  if (MI->isRegTiedToUseOperand(i)) continue;
1200 
1201  // Repeat for reg and all subregs.
1202  for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
1203  SubRegs.isValid(); ++SubRegs)
1204  LiveRegs.reset(*SubRegs);
1205  }
1206 
1207  // Examine all used registers and set/clear kill flag. When a
1208  // register is used multiple times we only set the kill flag on
1209  // the first use. Don't set kill flags on undef operands.
1210  killedRegs.reset();
1211  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
1212  MachineOperand &MO = MI->getOperand(i);
1213  if (!MO.isReg() || !MO.isUse() || MO.isUndef()) continue;
1214  unsigned Reg = MO.getReg();
1215  if ((Reg == 0) || MRI.isReserved(Reg)) continue;
1216 
1217  bool kill = false;
1218  if (!killedRegs.test(Reg)) {
1219  kill = true;
1220  // A register is not killed if any subregs are live...
1221  for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs) {
1222  if (LiveRegs.test(*SubRegs)) {
1223  kill = false;
1224  break;
1225  }
1226  }
1227 
1228  // If subreg is not live, then register is killed if it became
1229  // live in this instruction
1230  if (kill)
1231  kill = !LiveRegs.test(Reg);
1232  }
1233 
1234  if (MO.isKill() != kill) {
1235  DEBUG(dbgs() << "Fixing " << MO << " in ");
1236  // Warning: toggleKillFlag may invalidate MO.
1237  toggleKillFlag(MI, MO);
1238  DEBUG(MI->dump());
1239  DEBUG(if (MI->getOpcode() == TargetOpcode::BUNDLE) {
1242  while (++Begin != End)
1243  DEBUG(Begin->dump());
1244  });
1245  }
1246 
1247  killedRegs.set(Reg);
1248  }
1249 
1250  // Mark any used register (that is not using undef) and subregs as
1251  // now live...
1252  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
1253  MachineOperand &MO = MI->getOperand(i);
1254  if (!MO.isReg() || !MO.isUse() || MO.isUndef()) continue;
1255  unsigned Reg = MO.getReg();
1256  if ((Reg == 0) || MRI.isReserved(Reg)) continue;
1257 
1258  for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
1259  SubRegs.isValid(); ++SubRegs)
1260  LiveRegs.set(*SubRegs);
1261  }
1262  }
1263 }
1264 
1265 void ScheduleDAGInstrs::dumpNode(const SUnit *SU) const {
1266 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1267  SU->getInstr()->dump();
1268 #endif
1269 }
1270 
1271 std::string ScheduleDAGInstrs::getGraphNodeLabel(const SUnit *SU) const {
1272  std::string s;
1273  raw_string_ostream oss(s);
1274  if (SU == &EntrySU)
1275  oss << "<entry>";
1276  else if (SU == &ExitSU)
1277  oss << "<exit>";
1278  else
1279  SU->getInstr()->print(oss, /*SkipOpers=*/true);
1280  return oss.str();
1281 }
1282 
1283 /// Return the basic block label. It is not necessarilly unique because a block
1284 /// contains multiple scheduling regions. But it is fine for visualization.
1285 std::string ScheduleDAGInstrs::getDAGName() const {
1286  return "dag." + BB->getFullName();
1287 }
1288 
1289 //===----------------------------------------------------------------------===//
1290 // SchedDFSResult Implementation
1291 //===----------------------------------------------------------------------===//
1292 
1293 namespace llvm {
1294 /// \brief Internal state used to compute SchedDFSResult.
1296  SchedDFSResult &R;
1297 
1298  /// Join DAG nodes into equivalence classes by their subtree.
1299  IntEqClasses SubtreeClasses;
1300  /// List PredSU, SuccSU pairs that represent data edges between subtrees.
1301  std::vector<std::pair<const SUnit*, const SUnit*> > ConnectionPairs;
1302 
1303  struct RootData {
1304  unsigned NodeID;
1305  unsigned ParentNodeID; // Parent node (member of the parent subtree).
1306  unsigned SubInstrCount; // Instr count in this tree only, not children.
1307 
1308  RootData(unsigned id): NodeID(id),
1309  ParentNodeID(SchedDFSResult::InvalidSubtreeID),
1310  SubInstrCount(0) {}
1311 
1312  unsigned getSparseSetIndex() const { return NodeID; }
1313  };
1314 
1315  SparseSet<RootData> RootSet;
1316 
1317 public:
1318  SchedDFSImpl(SchedDFSResult &r): R(r), SubtreeClasses(R.DFSNodeData.size()) {
1319  RootSet.setUniverse(R.DFSNodeData.size());
1320  }
1321 
1322  /// Return true if this node been visited by the DFS traversal.
1323  ///
1324  /// During visitPostorderNode the Node's SubtreeID is assigned to the Node
1325  /// ID. Later, SubtreeID is updated but remains valid.
1326  bool isVisited(const SUnit *SU) const {
1327  return R.DFSNodeData[SU->NodeNum].SubtreeID
1328  != SchedDFSResult::InvalidSubtreeID;
1329  }
1330 
1331  /// Initialize this node's instruction count. We don't need to flag the node
1332  /// visited until visitPostorder because the DAG cannot have cycles.
1333  void visitPreorder(const SUnit *SU) {
1334  R.DFSNodeData[SU->NodeNum].InstrCount =
1335  SU->getInstr()->isTransient() ? 0 : 1;
1336  }
1337 
1338  /// Called once for each node after all predecessors are visited. Revisit this
1339  /// node's predecessors and potentially join them now that we know the ILP of
1340  /// the other predecessors.
1341  void visitPostorderNode(const SUnit *SU) {
1342  // Mark this node as the root of a subtree. It may be joined with its
1343  // successors later.
1344  R.DFSNodeData[SU->NodeNum].SubtreeID = SU->NodeNum;
1345  RootData RData(SU->NodeNum);
1346  RData.SubInstrCount = SU->getInstr()->isTransient() ? 0 : 1;
1347 
1348  // If any predecessors are still in their own subtree, they either cannot be
1349  // joined or are large enough to remain separate. If this parent node's
1350  // total instruction count is not greater than a child subtree by at least
1351  // the subtree limit, then try to join it now since splitting subtrees is
1352  // only useful if multiple high-pressure paths are possible.
1353  unsigned InstrCount = R.DFSNodeData[SU->NodeNum].InstrCount;
1355  PI = SU->Preds.begin(), PE = SU->Preds.end(); PI != PE; ++PI) {
1356  if (PI->getKind() != SDep::Data)
1357  continue;
1358  unsigned PredNum = PI->getSUnit()->NodeNum;
1359  if ((InstrCount - R.DFSNodeData[PredNum].InstrCount) < R.SubtreeLimit)
1360  joinPredSubtree(*PI, SU, /*CheckLimit=*/false);
1361 
1362  // Either link or merge the TreeData entry from the child to the parent.
1363  if (R.DFSNodeData[PredNum].SubtreeID == PredNum) {
1364  // If the predecessor's parent is invalid, this is a tree edge and the
1365  // current node is the parent.
1366  if (RootSet[PredNum].ParentNodeID == SchedDFSResult::InvalidSubtreeID)
1367  RootSet[PredNum].ParentNodeID = SU->NodeNum;
1368  }
1369  else if (RootSet.count(PredNum)) {
1370  // The predecessor is not a root, but is still in the root set. This
1371  // must be the new parent that it was just joined to. Note that
1372  // RootSet[PredNum].ParentNodeID may either be invalid or may still be
1373  // set to the original parent.
1374  RData.SubInstrCount += RootSet[PredNum].SubInstrCount;
1375  RootSet.erase(PredNum);
1376  }
1377  }
1378  RootSet[SU->NodeNum] = RData;
1379  }
1380 
1381  /// Called once for each tree edge after calling visitPostOrderNode on the
1382  /// predecessor. Increment the parent node's instruction count and
1383  /// preemptively join this subtree to its parent's if it is small enough.
1384  void visitPostorderEdge(const SDep &PredDep, const SUnit *Succ) {
1385  R.DFSNodeData[Succ->NodeNum].InstrCount
1386  += R.DFSNodeData[PredDep.getSUnit()->NodeNum].InstrCount;
1387  joinPredSubtree(PredDep, Succ);
1388  }
1389 
1390  /// Add a connection for cross edges.
1391  void visitCrossEdge(const SDep &PredDep, const SUnit *Succ) {
1392  ConnectionPairs.push_back(std::make_pair(PredDep.getSUnit(), Succ));
1393  }
1394 
1395  /// Set each node's subtree ID to the representative ID and record connections
1396  /// between trees.
1397  void finalize() {
1398  SubtreeClasses.compress();
1399  R.DFSTreeData.resize(SubtreeClasses.getNumClasses());
1400  assert(SubtreeClasses.getNumClasses() == RootSet.size()
1401  && "number of roots should match trees");
1403  RI = RootSet.begin(), RE = RootSet.end(); RI != RE; ++RI) {
1404  unsigned TreeID = SubtreeClasses[RI->NodeID];
1405  if (RI->ParentNodeID != SchedDFSResult::InvalidSubtreeID)
1406  R.DFSTreeData[TreeID].ParentTreeID = SubtreeClasses[RI->ParentNodeID];
1407  R.DFSTreeData[TreeID].SubInstrCount = RI->SubInstrCount;
1408  // Note that SubInstrCount may be greater than InstrCount if we joined
1409  // subtrees across a cross edge. InstrCount will be attributed to the
1410  // original parent, while SubInstrCount will be attributed to the joined
1411  // parent.
1412  }
1413  R.SubtreeConnections.resize(SubtreeClasses.getNumClasses());
1414  R.SubtreeConnectLevels.resize(SubtreeClasses.getNumClasses());
1415  DEBUG(dbgs() << R.getNumSubtrees() << " subtrees:\n");
1416  for (unsigned Idx = 0, End = R.DFSNodeData.size(); Idx != End; ++Idx) {
1417  R.DFSNodeData[Idx].SubtreeID = SubtreeClasses[Idx];
1418  DEBUG(dbgs() << " SU(" << Idx << ") in tree "
1419  << R.DFSNodeData[Idx].SubtreeID << '\n');
1420  }
1421  for (std::vector<std::pair<const SUnit*, const SUnit*> >::const_iterator
1422  I = ConnectionPairs.begin(), E = ConnectionPairs.end();
1423  I != E; ++I) {
1424  unsigned PredTree = SubtreeClasses[I->first->NodeNum];
1425  unsigned SuccTree = SubtreeClasses[I->second->NodeNum];
1426  if (PredTree == SuccTree)
1427  continue;
1428  unsigned Depth = I->first->getDepth();
1429  addConnection(PredTree, SuccTree, Depth);
1430  addConnection(SuccTree, PredTree, Depth);
1431  }
1432  }
1433 
1434 protected:
1435  /// Join the predecessor subtree with the successor that is its DFS
1436  /// parent. Apply some heuristics before joining.
1437  bool joinPredSubtree(const SDep &PredDep, const SUnit *Succ,
1438  bool CheckLimit = true) {
1439  assert(PredDep.getKind() == SDep::Data && "Subtrees are for data edges");
1440 
1441  // Check if the predecessor is already joined.
1442  const SUnit *PredSU = PredDep.getSUnit();
1443  unsigned PredNum = PredSU->NodeNum;
1444  if (R.DFSNodeData[PredNum].SubtreeID != PredNum)
1445  return false;
1446 
1447  // Four is the magic number of successors before a node is considered a
1448  // pinch point.
1449  unsigned NumDataSucs = 0;
1450  for (SUnit::const_succ_iterator SI = PredSU->Succs.begin(),
1451  SE = PredSU->Succs.end(); SI != SE; ++SI) {
1452  if (SI->getKind() == SDep::Data) {
1453  if (++NumDataSucs >= 4)
1454  return false;
1455  }
1456  }
1457  if (CheckLimit && R.DFSNodeData[PredNum].InstrCount > R.SubtreeLimit)
1458  return false;
1459  R.DFSNodeData[PredNum].SubtreeID = Succ->NodeNum;
1460  SubtreeClasses.join(Succ->NodeNum, PredNum);
1461  return true;
1462  }
1463 
1464  /// Called by finalize() to record a connection between trees.
1465  void addConnection(unsigned FromTree, unsigned ToTree, unsigned Depth) {
1466  if (!Depth)
1467  return;
1468 
1469  do {
1471  R.SubtreeConnections[FromTree];
1473  I = Connections.begin(), E = Connections.end(); I != E; ++I) {
1474  if (I->TreeID == ToTree) {
1475  I->Level = std::max(I->Level, Depth);
1476  return;
1477  }
1478  }
1479  Connections.push_back(SchedDFSResult::Connection(ToTree, Depth));
1480  FromTree = R.DFSTreeData[FromTree].ParentTreeID;
1481  } while (FromTree != SchedDFSResult::InvalidSubtreeID);
1482  }
1483 };
1484 } // namespace llvm
1485 
1486 namespace {
1487 /// \brief Manage the stack used by a reverse depth-first search over the DAG.
1488 class SchedDAGReverseDFS {
1489  std::vector<std::pair<const SUnit*, SUnit::const_pred_iterator> > DFSStack;
1490 public:
1491  bool isComplete() const { return DFSStack.empty(); }
1492 
1493  void follow(const SUnit *SU) {
1494  DFSStack.push_back(std::make_pair(SU, SU->Preds.begin()));
1495  }
1496  void advance() { ++DFSStack.back().second; }
1497 
1498  const SDep *backtrack() {
1499  DFSStack.pop_back();
1500  return DFSStack.empty() ? nullptr : std::prev(DFSStack.back().second);
1501  }
1502 
1503  const SUnit *getCurr() const { return DFSStack.back().first; }
1504 
1505  SUnit::const_pred_iterator getPred() const { return DFSStack.back().second; }
1506 
1507  SUnit::const_pred_iterator getPredEnd() const {
1508  return getCurr()->Preds.end();
1509  }
1510 };
1511 } // anonymous
1512 
1513 static bool hasDataSucc(const SUnit *SU) {
1515  SI = SU->Succs.begin(), SE = SU->Succs.end(); SI != SE; ++SI) {
1516  if (SI->getKind() == SDep::Data && !SI->getSUnit()->isBoundaryNode())
1517  return true;
1518  }
1519  return false;
1520 }
1521 
1522 /// Compute an ILP metric for all nodes in the subDAG reachable via depth-first
1523 /// search from this root.
1525  if (!IsBottomUp)
1526  llvm_unreachable("Top-down ILP metric is unimplemnted");
1527 
1528  SchedDFSImpl Impl(*this);
1530  SI = SUnits.begin(), SE = SUnits.end(); SI != SE; ++SI) {
1531  const SUnit *SU = &*SI;
1532  if (Impl.isVisited(SU) || hasDataSucc(SU))
1533  continue;
1534 
1535  SchedDAGReverseDFS DFS;
1536  Impl.visitPreorder(SU);
1537  DFS.follow(SU);
1538  for (;;) {
1539  // Traverse the leftmost path as far as possible.
1540  while (DFS.getPred() != DFS.getPredEnd()) {
1541  const SDep &PredDep = *DFS.getPred();
1542  DFS.advance();
1543  // Ignore non-data edges.
1544  if (PredDep.getKind() != SDep::Data
1545  || PredDep.getSUnit()->isBoundaryNode()) {
1546  continue;
1547  }
1548  // An already visited edge is a cross edge, assuming an acyclic DAG.
1549  if (Impl.isVisited(PredDep.getSUnit())) {
1550  Impl.visitCrossEdge(PredDep, DFS.getCurr());
1551  continue;
1552  }
1553  Impl.visitPreorder(PredDep.getSUnit());
1554  DFS.follow(PredDep.getSUnit());
1555  }
1556  // Visit the top of the stack in postorder and backtrack.
1557  const SUnit *Child = DFS.getCurr();
1558  const SDep *PredDep = DFS.backtrack();
1559  Impl.visitPostorderNode(Child);
1560  if (PredDep)
1561  Impl.visitPostorderEdge(*PredDep, DFS.getCurr());
1562  if (DFS.isComplete())
1563  break;
1564  }
1565  }
1566  Impl.finalize();
1567 }
1568 
1569 /// The root of the given SubtreeID was just scheduled. For all subtrees
1570 /// connected to this tree, record the depth of the connection so that the
1571 /// nearest connected subtrees can be prioritized.
1572 void SchedDFSResult::scheduleTree(unsigned SubtreeID) {
1574  I = SubtreeConnections[SubtreeID].begin(),
1575  E = SubtreeConnections[SubtreeID].end(); I != E; ++I) {
1576  SubtreeConnectLevels[I->TreeID] =
1577  std::max(SubtreeConnectLevels[I->TreeID], I->Level);
1578  DEBUG(dbgs() << " Tree: " << I->TreeID
1579  << " @" << SubtreeConnectLevels[I->TreeID] << '\n');
1580  }
1581 }
1582 
1584 void ILPValue::print(raw_ostream &OS) const {
1585  OS << InstrCount << " / " << Length << " = ";
1586  if (!Length)
1587  OS << "BADILP";
1588  else
1589  OS << format("%g", ((double)InstrCount / Length));
1590 }
1591 
1593 void ILPValue::dump() const {
1594  dbgs() << *this << '\n';
1595 }
1596 
1597 namespace llvm {
1598 
1601  Val.print(OS);
1602  return OS;
1603 }
1604 
1605 } // namespace llvm
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:192
iterator end()
Returns an iterator past this container.
SmallVector< PointerIntPair< ValueType, 1, bool >, 4 > UnderlyingObjectsVector
static void getUnderlyingObjectsForInstr(const MachineInstr *MI, const MachineFrameInfo *MFI, UnderlyingObjectsVector &Objects, const DataLayout &DL)
getUnderlyingObjectsForInstr - If this machine instr has memory reference information and it can be t...
const MachineFunction * getParent() const
getParent - Return the MachineFunction containing this basic block.
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:104
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:240
BitVector & set()
Definition: BitVector.h:218
virtual void finishBlock()
finishBlock - Clean up after scheduling in the given block.
static void adjustChainDeps(AliasAnalysis *AA, const MachineFrameInfo *MFI, const DataLayout &DL, SUnit *SU, SUnit *ExitSU, std::set< SUnit * > &CheckList, unsigned LatencyToLoad)
This function assumes that "downward" from SU there exist tail/leaf of already constructed DAG...
bool joinPredSubtree(const SDep &PredDep, const SUnit *Succ, bool CheckLimit=true)
Join the predecessor subtree with the successor that is its DFS parent.
MachineBasicBlock::instr_iterator getBundleEnd(MachineInstr *MI)
Return an iterator pointing beyond the bundle containing MI.
void clear()
Definition: MapVector.h:65
PointerUnion< const Value *, const PseudoSourceValue * > ValueType
SlotIndex def
The index of the defining instruction.
Definition: LiveInterval.h:53
iterator insert(const ValueT &Val)
Insert a new element at the tail of the subset list.
bool isSucc(SUnit *N)
isSucc - Test if node N is a successor of this node.
Definition: ScheduleDAG.h:466
bool contains(const KeyT &Key) const
Returns true if this set contains an element identified by Key.
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
Definition: Compiler.h:344
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
void init(unsigned N)
Initialize an array of N PressureDiffs.
Record a physical register access.
int getNumber() const
getNumber - MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a M...
std::pair< iterator, bool > insert(const ValueT &Val)
insert - Attempts to insert a new element.
Definition: SparseSet.h:249
void addVRegDefDeps(SUnit *SU, unsigned OperIdx)
addVRegDefDeps - Add register output and data dependencies from this SUnit to instructions that occur...
unsigned computeOutputLatency(const MachineInstr *DefMI, unsigned DefIdx, const MachineInstr *DepMI) const
Output dependency latency of a pair of defs of the same register.
void buildSchedGraph(AliasAnalysis *AA, RegPressureTracker *RPTracker=nullptr, PressureDiffs *PDiffs=nullptr)
buildSchedGraph - Build SUnits from the MachineBasicBlock that we are input.
void dumpNode(const SUnit *SU) const override
size_type count(const KeyT &Key) const
count - Returns 1 if this set contains an element identified by Key, 0 otherwise. ...
Definition: SparseSet.h:235
std::vector< unsigned >::const_iterator livein_iterator
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition: MachineInstr.h:579
MachineInstr * getInstr() const
getInstr - Return the representative MachineInstr for this SUnit.
Definition: ScheduleDAG.h:406
iterator end() const
Definition: ArrayRef.h:123
bool isDead() const
SlotIndex getInstructionIndex(const MachineInstr *instr) const
Returns the base index of the given instruction.
Represent the ILP of the subDAG rooted at a DAG node.
Definition: ScheduleDFS.h:35
TargetSchedModel SchedModel
TargetSchedModel provides an interface to the machine model.
void init(const MCSchedModel &sm, const TargetSubtargetInfo *sti, const TargetInstrInfo *tii)
Initialize the machine model for instruction scheduling.
void compress()
compress - Compress equivalence classes by numbering them 0 .
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
bool CanHandleTerminators
The standard DAG builder does not normally include terminators as DAG nodes because it does not creat...
MachineBasicBlock::iterator begin() const
begin - Return an iterator to the top of the current scheduling region.
const_iterator begin(StringRef path)
Get begin iterator over path.
Definition: Path.cpp:232
This class implements a map that also provides access to all stored values in a deterministic order...
Definition: MapVector.h:32
const MCSchedClassDesc * getSchedClass(SUnit *SU) const
Resolve and cache a resolved scheduling class for an SUnit.
The two locations do not alias at all.
Definition: AliasAnalysis.h:78
static cl::opt< bool > EnableAASchedMI("enable-aa-sched-mi", cl::Hidden, cl::ZeroOrMore, cl::init(false), cl::desc("Enable use of AA during MI DAG construction"))
void GetUnderlyingObjects(Value *V, SmallVectorImpl< Value * > &Objects, const DataLayout &DL, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to GetUnderlyingObject except that it can look through phi and select instruct...
Kind
Kind - These are the different kinds of scheduling dependencies.
Definition: ScheduleDAG.h:48
VNInfo - Value Number Information.
Definition: LiveInterval.h:45
Instructions::iterator instr_iterator
unsigned NumRegionInstrs
Instructions in this region (distance(RegionBegin, RegionEnd)).
void addPhysRegDataDeps(SUnit *SU, unsigned OperIdx)
MO is an operand of SU's instruction that defines a physical register.
void fixupKills(MachineBasicBlock *MBB)
Fix register kill flags that scheduling has made invalid.
RangePair equal_range(const KeyT &K)
The bounds of the range of items sharing Key K.
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
Definition: MachineInstr.h:419
SmallVector< SDep, 4 > Preds
Definition: ScheduleDAG.h:275
virtual void startBlock(MachineBasicBlock *BB)
startBlock - Prepare to perform scheduling in the given block.
The two locations may or may not alias. This is the least precise result.
Definition: AliasAnalysis.h:80
MachineBasicBlock::const_iterator getPos() const
Get the MI position corresponding to this register pressure.
bool empty() const
empty - Returns true if the set is empty.
Definition: SparseSet.h:183
A register anti-dependedence (aka WAR).
Definition: ScheduleDAG.h:50
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:242
MachineFunction & MF
Definition: ScheduleDAG.h:563
unsigned getNumSubtrees() const
The number of subtrees detected in this DAG.
Definition: ScheduleDFS.h:164
std::vector< SUnit * > PendingLoads
PendingLoads - Remember where unknown loads are after the most recent unknown store, as we iterate.
unsigned getNumVirtRegs() const
getNumVirtRegs - Return the number of virtual registers created.
MachineMemOperand - A description of a memory reference used in the backend.
unsigned NumSuccs
Definition: ScheduleDAG.h:286
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineBasicBlock::iterator RegionEnd
The end of the range to be scheduled.
DenseMap< MachineInstr *, SUnit * > MISUnitMap
After calling BuildSchedGraph, each machine instruction in the current scheduling region is mapped to...
void addSchedBarrierDeps()
addSchedBarrierDeps - Add dependencies from instructions in the current list of instructions being sc...
const HexagonInstrInfo * TII
static void getUnderlyingObjects(const Value *V, SmallVectorImpl< Value * > &Objects, const DataLayout &DL)
getUnderlyingObjects - This is a wrapper around GetUnderlyingObjects and adds support for basic ptrto...
An individual mapping from virtual register number to SUnit.
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
Definition: SmallVector.h:406
void setInstr(MachineInstr *MI)
setInstr - Assign the instruction for the SUnit.
Definition: ScheduleDAG.h:399
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
static bool hasDataSucc(const SUnit *SU)
static void advance(T &it, size_t Val)
virtual bool areMemAccessesTriviallyDisjoint(MachineInstr *MIa, MachineInstr *MIb, AliasAnalysis *AA=nullptr) const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Regular data dependence (aka true-dependence).
Definition: ScheduleDAG.h:49
const_iterator end() const
Definition: SparseSet.h:175
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
Definition: MachineInstr.h:566
Result of a LiveRange query.
Definition: LiveInterval.h:86
bool hasPhysRegUses
Definition: ScheduleDAG.h:298
std::vector< MachineBasicBlock * >::iterator succ_iterator
Reg
All possible values of the reg field in the ModR/M byte.
bool hasPhysRegDefs
Definition: ScheduleDAG.h:299
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
bool isUndef() const
const MCSchedModel & getSchedModel() const
Get the machine model for this subtarget's CPU.
#define false
Definition: ConvertUTF.c:65
const MachineFrameInfo * MFI
unsigned getNumOperands() const
Access to explicit operands of the instruction.
Definition: MachineInstr.h:271
bool isIdentifiedObject(const Value *V)
isIdentifiedObject - Return true if this pointer refers to a distinct and identifiable object...
Compute the values of each DAG node for various metrics during DFS.
Definition: ScheduleDFS.h:66
unsigned getNumRegs() const
Return the number of registers this target has (useful for sizing arrays holding per register informa...
A register output-dependence (aka WAW).
Definition: ScheduleDAG.h:51
bool isKill() const
bool hasReservedResource
Definition: ScheduleDAG.h:308
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
MachineBasicBlock::iterator RegionBegin
The beginning of the range to be scheduled.
void addVRegUseDeps(SUnit *SU, unsigned OperIdx)
addVRegUseDeps - Add a register data dependency if the instruction that defines the virtual register ...
void print(raw_ostream &OS) const
bool recede(SmallVectorImpl< unsigned > *LiveUses=nullptr, PressureDiff *PDiff=nullptr)
Recede across the previous instruction.
VReg2UseMap VRegUses
After calling BuildSchedGraph, each vreg used in the scheduling region is mapped to a set of SUnits...
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: ArrayRef.h:31
virtual void adjustSchedDependency(SUnit *def, SUnit *use, SDep &dep) const
bool IsPostRA
isPostRA flag indicates vregs cannot be present.
virtual void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs)
Initialize the scheduler state for the next scheduling region.
void visitPostorderNode(const SUnit *SU)
Called once for each node after all predecessors are visited.
iterator find(const KeyT &Key)
Definition: MapVector.h:108
void visitPreorder(const SUnit *SU)
Initialize this node's instruction count.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:267
iterator erase(iterator I)
erase - Erases an existing element identified by a valid iterator.
Definition: SparseSet.h:280
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:120
format_object< Ts...> format(const char *Fmt, const Ts &...Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:111
TargetInstrInfo - Interface to description of machine instruction set.
bool isDebugValue() const
Definition: MachineInstr.h:748
static void toggleBundleKillFlag(MachineInstr *MI, unsigned Reg, bool NewKillState)
If we change a kill flag on the bundle instruction implicit register operands, then we also need to p...
SDep - Scheduling dependency.
Definition: ScheduleDAG.h:45
bool isUnbuffered
Definition: ScheduleDAG.h:307
bundle_iterator< MachineInstr, instr_iterator > iterator
#define P(N)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
void clearBitsNotInMask(const uint32_t *Mask, unsigned MaskWords=~0u)
clearBitsNotInMask - Clear a bit in this vector for every '0' bit in Mask.
Definition: BitVector.h:499
LiveQueryResult Query(SlotIndex Idx) const
Query Liveness at Idx.
Definition: LiveInterval.h:493
Array of PressureDiffs.
size_type size() const
size - Returns the number of elements in the set.
Definition: SparseSet.h:190
void clearDAG()
clearDAG - clear the DAG state (between regions).
Definition: ScheduleDAG.cpp:50
unsigned short Latency
Definition: ScheduleDAG.h:292
Identify one of the processor resource kinds consumed by a particular scheduling class for the specif...
Definition: MCSchedule.h:55
void print(raw_ostream &OS, bool SkipOpers=false) const
AAMDNodes getAAInfo() const
getAAInfo - Return the AA tags for the memory reference.
bool RemoveKillFlags
True if the DAG builder should remove kill flags (in preparation for rescheduling).
bool isReserved(unsigned PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
void visitCrossEdge(const SDep &PredDep, const SUnit *Succ)
Add a connection for cross edges.
Internal state used to compute SchedDFSResult.
int findRegisterDefOperandIdx(unsigned Reg, bool isDead=false, bool Overlap=false, const TargetRegisterInfo *TRI=nullptr) const
Returns the operand index that is a def of the specified register or -1 if it is not found...
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:273
Summarize the scheduling resources required for an instruction of a particular scheduling class...
Definition: MCSchedule.h:101
void addConnection(unsigned FromTree, unsigned ToTree, unsigned Depth)
Called by finalize() to record a connection between trees.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:264
bool isRegTiedToUseOperand(unsigned DefOpIdx, unsigned *UseOpIdx=nullptr) const
Given the index of a register def operand, check if the register def is tied to a source operand...
Definition: MachineInstr.h:995
void setUniverse(unsigned U)
Set the universe size which determines the largest key the set can hold.
AliasResult
The possible results of an alias query.
Definition: AliasAnalysis.h:72
MCRegAliasIterator enumerates all registers aliasing Reg.
BitVector & reset()
Definition: BitVector.h:259
bool isPosition() const
Definition: MachineInstr.h:746
size_type size() const
Definition: MapVector.h:44
Track the current register pressure at some position in the instruction stream, and remember the high...
iterator begin() const
Definition: ArrayRef.h:122
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
Definition: MachineInstr.h:352
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore...
VReg2SUnitMap VRegDefs
Track the last instruction in this region defining each virtual register.
bool registerDefIsDead(unsigned Reg, const TargetRegisterInfo *TRI=nullptr) const
Returns true if the register is dead in this machine instruction.
Definition: MachineInstr.h:881
List of PressureChanges in order of increasing, unique PSetID.
virtual void exitRegion()
Notify that the scheduler has finished scheduling the current region.
unsigned InstrCount
Definition: ScheduleDFS.h:36
void setUniverse(unsigned U)
setUniverse - Set the universe size which determines the largest key the set can hold.
Definition: SparseSet.h:155
bool isInvariantLoad(AliasAnalysis *AA) const
Return true if this instruction is loading from a location whose value is invariant across the functi...
const TargetMachine & TM
Definition: ScheduleDAG.h:560
void clear()
Clears the set.
static bool MIsNeedChainEdge(AliasAnalysis *AA, const MachineFrameInfo *MFI, const DataLayout &DL, MachineInstr *MIa, MachineInstr *MIb)
This returns true if the two MIs need a chain edge betwee them.
void join(unsigned a, unsigned b)
join - Join the equivalence classes of a and b.
bool isPointerTy() const
isPointerTy - True if this is an instance of PointerType.
Definition: Type.h:217
MCSubRegIterator enumerates all sub-registers of Reg.
std::string & str()
Flushes the stream contents to the target string and returns the string's reference.
Definition: raw_ostream.h:480
std::string getDAGName() const override
Return a label for the region of code covered by the DAG.
bool isVisited(const SUnit *SU) const
Return true if this node been visited by the DFS traversal.
const_iterator begin() const
Definition: SparseSet.h:174
ProcResIter getWriteProcResEnd(const MCSchedClassDesc *SC) const
void setIsKill(bool Val=true)
iterator find(const KeyT &Key)
Find an element by its key.
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
An unknown scheduling barrier.
Definition: ScheduleDAG.h:65
std::string getFullName() const
getFullName - Return a formatted string to identify this block and its parent function.
void addPhysRegDeps(SUnit *SU, unsigned OperIdx)
addPhysRegDeps - Add register dependencies (data, anti, and output) from this SUnit to following inst...
bool hasInstrSchedModel() const
Return true if this machine model includes an instruction-level scheduling model. ...
Representation for a specific memory location.
std::string getGraphNodeLabel(const SUnit *SU) const override
Return a label for a DAG node that points to an instruction.
bool memoperands_empty() const
Definition: MachineInstr.h:342
bool isCommutable
Definition: ScheduleDAG.h:297
bool toggleKillFlag(MachineInstr *MI, MachineOperand &MO)
Toggle a register operand kill flag.
bool hasTailCall() const
Returns true if the function contains a tail call.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:299
const MCProcResourceDesc * getProcResource(unsigned PIdx) const
Get a processor resource by ID for convenience.
const uint32_t * getRegMask() const
getRegMask - Returns a bit mask of registers preserved by this RegMask operand.
void compute(ArrayRef< SUnit > SUnits)
Compute various metrics for the DAG with given roots.
const DataLayout * getDataLayout() const
Deprecated in 3.7, will be removed in 3.8.
virtual AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB)
Alias Queries...
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
This is a utility class that provides an abstraction for the common functionality between Instruction...
Definition: Operator.h:32
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
bool test(unsigned Idx) const
Definition: BitVector.h:322
Reg2SUnitsMap Defs
State internal to DAG building.
void eraseAll(const KeyT &K)
Erase all elements with the given key.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:548
CHAIN = SC CHAIN, Imm128 - System call.
static unsigned iterateChainSucc(AliasAnalysis *AA, const MachineFrameInfo *MFI, const DataLayout &DL, SUnit *SUa, SUnit *SUb, SUnit *ExitSU, unsigned *Depth, SmallPtrSetImpl< const SUnit * > &Visited)
This recursive function iterates over chain deps of SUb looking for "latest" node that needs a chain ...
Nonvolatile load/Store instructions that may alias.
Definition: ScheduleDAG.h:66
LiveInterval & getInterval(unsigned Reg)
bool isTransient() const
Return true if this is a transient instruction that is either very likely to be eliminated during reg...
Definition: MachineInstr.h:804
unsigned Length
Length may either correspond to depth or height, depending on direction, and cycles or nodes dependin...
Definition: ScheduleDFS.h:39
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
BUNDLE - This instruction represents an instruction bundle.
Definition: TargetOpcodes.h:91
void dump() const
const Value * getValue() const
getValue - Return the base address of the memory access.
PseudoSourceValue - Special value supplied for machine level alias analysis.
SUnit * getSUnit() const
Definition: ScheduleDAG.h:160
bool isIntegerTy() const
isIntegerTy - True if this is an instance of IntegerType.
Definition: Type.h:193
MachineBasicBlock::iterator end() const
end - Return an iterator to the bottom of the current scheduling region.
ScheduleDAGInstrs(MachineFunction &mf, const MachineLoopInfo *mli, bool IsPostRAFlag, bool RemoveKillFlags=false, LiveIntervals *LIS=nullptr)
bool isBoundaryNode() const
Boundary nodes are placeholders for the boundary of the scheduling region.
Definition: ScheduleDAG.h:377
void visitPostorderEdge(const SDep &PredDep, const SUnit *Succ)
Called once for each tree edge after calling visitPostOrderNode on the predecessor.
void setLatency(unsigned Lat)
setLatency - Set the latency for this edge.
Definition: ScheduleDAG.h:155
**iterator erase(iterator I)
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
Definition: Operator.h:48
TargetSubtargetInfo - Generic base class for all target subtargets.
SUnit * getSUnit(MachineInstr *MI) const
getSUnit - Return an existing SUnit for this MI, or NULL.
Representation of each machine instruction.
Definition: MachineInstr.h:51
void finalize()
Set each node's subtree ID to the representative ID and record connections between trees...
static bool isPhysicalRegister(unsigned Reg)
isPhysicalRegister - Return true if the specified register number is in the physical register namespa...
bool hasOneDef(unsigned RegNo) const
hasOneDef - Return true if there is exactly one instruction defining the specified register...
const TargetRegisterInfo * TRI
Definition: ScheduleDAG.h:562
static cl::opt< bool > UseTBAA("use-tbaa-in-sched-mi", cl::Hidden, cl::init(true), cl::desc("Enable use of TBAA during MI DAG construction"))
iterator find(const KeyT &Key)
find - Find an element by its key.
Definition: SparseSet.h:224
#define I(x, y, z)
Definition: MD5.cpp:54
bool isCall(QueryType Type=AnyInBundle) const
Definition: MachineInstr.h:403
static void addChainDependency(AliasAnalysis *AA, const MachineFrameInfo *MFI, const DataLayout &DL, SUnit *SUa, SUnit *SUb, std::set< SUnit * > &RejectList, unsigned TrueMemOrderLatency=0, bool isNormalMemory=false)
Check whether two objects need a chain edge, if so, add it otherwise remember the rejected SU...
void size_t size
Kind getKind() const
getKind - Return an enum value representing the kind of the dependence.
Definition: ScheduleDAG.h:170
const TargetInstrInfo * TII
Definition: ScheduleDAG.h:561
raw_ostream & operator<<(raw_ostream &OS, const APInt &I)
Definition: APInt.h:1738
unsigned computeOperandLatency(const MachineInstr *DefMI, unsigned DefOperIdx, const MachineInstr *UseMI, unsigned UseOperIdx) const
Compute operand latency based on the available machine model.
void clear()
clear - Clears the set.
Definition: SparseSet.h:194
unsigned NodeNum
Definition: ScheduleDAG.h:283
iterator begin()
Definition: MapVector.h:46
unsigned getReg() const
getReg - Returns the register number.
SUnit * newSUnit(MachineInstr *MI)
newSUnit - Creates a new SUnit and return a ptr to it.
bool isCommutable(QueryType Type=IgnoreBundle) const
Return true if this may be a 2- or 3-address instruction (of the form "X = op Y, Z, ..."), which produces the same result if Y and Z are exchanged.
Definition: MachineInstr.h:607
iterator end()
Definition: MapVector.h:48
const ARM::ArchExtKind Kind
bool addPred(const SDep &D, bool Required=true)
addPred - This adds the specified edge as a pred of the current node if not already.
Definition: ScheduleDAG.cpp:65
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:465
void startBlockForKills(MachineBasicBlock *BB)
PostRA helper for rewriting kill flags.
virtual const TargetInstrInfo * getInstrInfo() const
LLVM Value Representation.
Definition: Value.h:69
int64_t getOffset() const
getOffset - For normal values, this is a byte offset added to the base address.
void initSUnits()
Create an SUnit for each real instruction, numbered in top-down toplological order.
unsigned getNumClasses() const
getNumClasses - Return the number of equivalence classes after compress() was called.
Definition: IntEqClasses.h:72
SmallVector< SDep, 4 > Succs
Definition: ScheduleDAG.h:276
MachineInstr * getInstructionFromIndex(SlotIndex index) const
Returns the instruction associated with the given index.
Arbitrary strong DAG edge (no real dependence).
Definition: ScheduleDAG.h:68
void scheduleTree(unsigned SubtreeID)
Scheduler callback to update SubtreeConnectLevels when a tree is initially scheduled.
uint64_t getSize() const
getSize - Return the size in bytes of the memory reference.
This class implements an extremely fast bulk output stream that can only output to a stream...
Definition: raw_ostream.h:38
#define DEBUG(X)
Definition: Debug.h:92
static bool isGlobalMemoryObject(AliasAnalysis *AA, MachineInstr *MI)
Return true if MI is an instruction we are unable to reason about (like a call or something with unmo...
MachineBasicBlock * BB
State specific to the current scheduling region.
bool empty() const
Returns true if the set is empty.
static bool isUnsafeMemoryObject(MachineInstr *MI, const MachineFrameInfo *MFI, const DataLayout &DL)
bool readsReg() const
readsReg - Returns true if this operand reads the previous value of its register. ...
VNInfo * valueIn() const
Return the value that is live-in to the instruction.
Definition: LiveInterval.h:101
static bool isVolatile(Instruction *Inst)
MachineRegisterInfo & MRI
Definition: ScheduleDAG.h:564
std::vector< SUnit > SUnits
Definition: ScheduleDAG.h:565
SchedDFSImpl(SchedDFSResult &r)
ProcResIter getWriteProcResBegin(const MCSchedClassDesc *SC) const
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
addReg - Add a new virtual register operand...
static const Value * getUnderlyingObjectFromInt(const Value *V)
getUnderlyingObjectFromInt - This is the function that does the work of looking through basic ptrtoin...
BitVector LiveRegs
Set of live physical registers for updating kill flags.
bool isBarrier(QueryType Type=AnyInBundle) const
Returns true if the specified instruction stops control flow from executing the instruction immediate...
Definition: MachineInstr.h:410
LiveIntervals * LIS
Live Intervals provides reaching defs in preRA scheduling.
SUnit - Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:261
virtual bool useAA() const
Enable use of alias analysis during code generation (during MI scheduling, DAGCombine, etc.).
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:340
void resize(size_type N)
Definition: SmallVector.h:376