File: | lib/CodeGen/ScheduleDAGInstrs.cpp |
Location: | line 799, column 9 |
Description: | Called C++ object pointer is null |
1 | //===---- ScheduleDAGInstrs.cpp - MachineInstr Rescheduling ---------------===// | |||
2 | // | |||
3 | // The LLVM Compiler Infrastructure | |||
4 | // | |||
5 | // This file is distributed under the University of Illinois Open Source | |||
6 | // License. See LICENSE.TXT for details. | |||
7 | // | |||
8 | //===----------------------------------------------------------------------===// | |||
9 | // | |||
10 | // This implements the ScheduleDAGInstrs class, which implements re-scheduling | |||
11 | // of MachineInstrs. | |||
12 | // | |||
13 | //===----------------------------------------------------------------------===// | |||
14 | ||||
15 | #include "llvm/CodeGen/ScheduleDAGInstrs.h" | |||
16 | #include "llvm/ADT/MapVector.h" | |||
17 | #include "llvm/ADT/SmallPtrSet.h" | |||
18 | #include "llvm/ADT/SmallSet.h" | |||
19 | #include "llvm/Analysis/AliasAnalysis.h" | |||
20 | #include "llvm/Analysis/ValueTracking.h" | |||
21 | #include "llvm/CodeGen/LiveIntervalAnalysis.h" | |||
22 | #include "llvm/CodeGen/MachineFunctionPass.h" | |||
23 | #include "llvm/CodeGen/MachineFrameInfo.h" | |||
24 | #include "llvm/CodeGen/MachineInstrBuilder.h" | |||
25 | #include "llvm/CodeGen/MachineMemOperand.h" | |||
26 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||
27 | #include "llvm/CodeGen/PseudoSourceValue.h" | |||
28 | #include "llvm/CodeGen/RegisterPressure.h" | |||
29 | #include "llvm/CodeGen/ScheduleDFS.h" | |||
30 | #include "llvm/IR/Operator.h" | |||
31 | #include "llvm/Support/CommandLine.h" | |||
32 | #include "llvm/Support/Debug.h" | |||
33 | #include "llvm/Support/Format.h" | |||
34 | #include "llvm/Support/raw_ostream.h" | |||
35 | #include "llvm/Target/TargetInstrInfo.h" | |||
36 | #include "llvm/Target/TargetMachine.h" | |||
37 | #include "llvm/Target/TargetRegisterInfo.h" | |||
38 | #include "llvm/Target/TargetSubtargetInfo.h" | |||
39 | #include <queue> | |||
40 | ||||
41 | using namespace llvm; | |||
42 | ||||
43 | #define DEBUG_TYPE"misched" "misched" | |||
44 | ||||
45 | static cl::opt<bool> EnableAASchedMI("enable-aa-sched-mi", cl::Hidden, | |||
46 | cl::ZeroOrMore, cl::init(false), | |||
47 | cl::desc("Enable use of AA during MI DAG construction")); | |||
48 | ||||
49 | static cl::opt<bool> UseTBAA("use-tbaa-in-sched-mi", cl::Hidden, | |||
50 | cl::init(true), cl::desc("Enable use of TBAA during MI DAG construction")); | |||
51 | ||||
52 | ScheduleDAGInstrs::ScheduleDAGInstrs(MachineFunction &mf, | |||
53 | const MachineLoopInfo *mli, | |||
54 | LiveIntervals *LIS, | |||
55 | bool RemoveKillFlags) | |||
56 | : ScheduleDAG(mf), MLI(mli), MFI(mf.getFrameInfo()), LIS(LIS), | |||
57 | RemoveKillFlags(RemoveKillFlags), CanHandleTerminators(false), | |||
58 | FirstDbgValue(nullptr) { | |||
59 | DbgValues.clear(); | |||
60 | ||||
61 | const TargetSubtargetInfo &ST = mf.getSubtarget(); | |||
62 | SchedModel.init(ST.getSchedModel(), &ST, TII); | |||
63 | } | |||
64 | ||||
65 | /// getUnderlyingObjectFromInt - This is the function that does the work of | |||
66 | /// looking through basic ptrtoint+arithmetic+inttoptr sequences. | |||
67 | static const Value *getUnderlyingObjectFromInt(const Value *V) { | |||
68 | do { | |||
69 | if (const Operator *U = dyn_cast<Operator>(V)) { | |||
70 | // If we find a ptrtoint, we can transfer control back to the | |||
71 | // regular getUnderlyingObjectFromInt. | |||
72 | if (U->getOpcode() == Instruction::PtrToInt) | |||
73 | return U->getOperand(0); | |||
74 | // If we find an add of a constant, a multiplied value, or a phi, it's | |||
75 | // likely that the other operand will lead us to the base | |||
76 | // object. We don't have to worry about the case where the | |||
77 | // object address is somehow being computed by the multiply, | |||
78 | // because our callers only care when the result is an | |||
79 | // identifiable object. | |||
80 | if (U->getOpcode() != Instruction::Add || | |||
81 | (!isa<ConstantInt>(U->getOperand(1)) && | |||
82 | Operator::getOpcode(U->getOperand(1)) != Instruction::Mul && | |||
83 | !isa<PHINode>(U->getOperand(1)))) | |||
84 | return V; | |||
85 | V = U->getOperand(0); | |||
86 | } else { | |||
87 | return V; | |||
88 | } | |||
89 | assert(V->getType()->isIntegerTy() && "Unexpected operand type!")((V->getType()->isIntegerTy() && "Unexpected operand type!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntegerTy() && \"Unexpected operand type!\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 89, __PRETTY_FUNCTION__)); | |||
90 | } while (1); | |||
91 | } | |||
92 | ||||
93 | /// getUnderlyingObjects - This is a wrapper around GetUnderlyingObjects | |||
94 | /// and adds support for basic ptrtoint+arithmetic+inttoptr sequences. | |||
95 | static void getUnderlyingObjects(const Value *V, | |||
96 | SmallVectorImpl<Value *> &Objects, | |||
97 | const DataLayout &DL) { | |||
98 | SmallPtrSet<const Value *, 16> Visited; | |||
99 | SmallVector<const Value *, 4> Working(1, V); | |||
100 | do { | |||
101 | V = Working.pop_back_val(); | |||
102 | ||||
103 | SmallVector<Value *, 4> Objs; | |||
104 | GetUnderlyingObjects(const_cast<Value *>(V), Objs, DL); | |||
105 | ||||
106 | for (SmallVectorImpl<Value *>::iterator I = Objs.begin(), IE = Objs.end(); | |||
107 | I != IE; ++I) { | |||
108 | V = *I; | |||
109 | if (!Visited.insert(V).second) | |||
110 | continue; | |||
111 | if (Operator::getOpcode(V) == Instruction::IntToPtr) { | |||
112 | const Value *O = | |||
113 | getUnderlyingObjectFromInt(cast<User>(V)->getOperand(0)); | |||
114 | if (O->getType()->isPointerTy()) { | |||
115 | Working.push_back(O); | |||
116 | continue; | |||
117 | } | |||
118 | } | |||
119 | Objects.push_back(const_cast<Value *>(V)); | |||
120 | } | |||
121 | } while (!Working.empty()); | |||
122 | } | |||
123 | ||||
124 | typedef PointerUnion<const Value *, const PseudoSourceValue *> ValueType; | |||
125 | typedef SmallVector<PointerIntPair<ValueType, 1, bool>, 4> | |||
126 | UnderlyingObjectsVector; | |||
127 | ||||
128 | /// getUnderlyingObjectsForInstr - If this machine instr has memory reference | |||
129 | /// information and it can be tracked to a normal reference to a known | |||
130 | /// object, return the Value for that object. | |||
131 | static void getUnderlyingObjectsForInstr(const MachineInstr *MI, | |||
132 | const MachineFrameInfo *MFI, | |||
133 | UnderlyingObjectsVector &Objects, | |||
134 | const DataLayout &DL) { | |||
135 | if (!MI->hasOneMemOperand() || | |||
136 | (!(*MI->memoperands_begin())->getValue() && | |||
137 | !(*MI->memoperands_begin())->getPseudoValue()) || | |||
138 | (*MI->memoperands_begin())->isVolatile()) | |||
139 | return; | |||
140 | ||||
141 | if (const PseudoSourceValue *PSV = | |||
142 | (*MI->memoperands_begin())->getPseudoValue()) { | |||
143 | // Function that contain tail calls don't have unique PseudoSourceValue | |||
144 | // objects. Two PseudoSourceValues might refer to the same or overlapping | |||
145 | // locations. The client code calling this function assumes this is not the | |||
146 | // case. So return a conservative answer of no known object. | |||
147 | if (MFI->hasTailCall()) | |||
148 | return; | |||
149 | ||||
150 | // For now, ignore PseudoSourceValues which may alias LLVM IR values | |||
151 | // because the code that uses this function has no way to cope with | |||
152 | // such aliases. | |||
153 | if (!PSV->isAliased(MFI)) { | |||
154 | bool MayAlias = PSV->mayAlias(MFI); | |||
155 | Objects.push_back(UnderlyingObjectsVector::value_type(PSV, MayAlias)); | |||
156 | } | |||
157 | return; | |||
158 | } | |||
159 | ||||
160 | const Value *V = (*MI->memoperands_begin())->getValue(); | |||
161 | if (!V) | |||
162 | return; | |||
163 | ||||
164 | SmallVector<Value *, 4> Objs; | |||
165 | getUnderlyingObjects(V, Objs, DL); | |||
166 | ||||
167 | for (Value *V : Objs) { | |||
168 | if (!isIdentifiedObject(V)) { | |||
169 | Objects.clear(); | |||
170 | return; | |||
171 | } | |||
172 | ||||
173 | Objects.push_back(UnderlyingObjectsVector::value_type(V, true)); | |||
174 | } | |||
175 | } | |||
176 | ||||
177 | void ScheduleDAGInstrs::startBlock(MachineBasicBlock *bb) { | |||
178 | BB = bb; | |||
179 | } | |||
180 | ||||
181 | void ScheduleDAGInstrs::finishBlock() { | |||
182 | // Subclasses should no longer refer to the old block. | |||
183 | BB = nullptr; | |||
184 | } | |||
185 | ||||
186 | /// Initialize the DAG and common scheduler state for the current scheduling | |||
187 | /// region. This does not actually create the DAG, only clears it. The | |||
188 | /// scheduling driver may call BuildSchedGraph multiple times per scheduling | |||
189 | /// region. | |||
190 | void ScheduleDAGInstrs::enterRegion(MachineBasicBlock *bb, | |||
191 | MachineBasicBlock::iterator begin, | |||
192 | MachineBasicBlock::iterator end, | |||
193 | unsigned regioninstrs) { | |||
194 | assert(bb == BB && "startBlock should set BB")((bb == BB && "startBlock should set BB") ? static_cast <void> (0) : __assert_fail ("bb == BB && \"startBlock should set BB\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 194, __PRETTY_FUNCTION__)); | |||
195 | RegionBegin = begin; | |||
196 | RegionEnd = end; | |||
197 | NumRegionInstrs = regioninstrs; | |||
198 | } | |||
199 | ||||
200 | /// Close the current scheduling region. Don't clear any state in case the | |||
201 | /// driver wants to refer to the previous scheduling region. | |||
202 | void ScheduleDAGInstrs::exitRegion() { | |||
203 | // Nothing to do. | |||
204 | } | |||
205 | ||||
206 | /// addSchedBarrierDeps - Add dependencies from instructions in the current | |||
207 | /// list of instructions being scheduled to scheduling barrier by adding | |||
208 | /// the exit SU to the register defs and use list. This is because we want to | |||
209 | /// make sure instructions which define registers that are either used by | |||
210 | /// the terminator or are live-out are properly scheduled. This is | |||
211 | /// especially important when the definition latency of the return value(s) | |||
212 | /// are too high to be hidden by the branch or when the liveout registers | |||
213 | /// used by instructions in the fallthrough block. | |||
214 | void ScheduleDAGInstrs::addSchedBarrierDeps() { | |||
215 | MachineInstr *ExitMI = RegionEnd != BB->end() ? &*RegionEnd : nullptr; | |||
216 | ExitSU.setInstr(ExitMI); | |||
217 | bool AllDepKnown = ExitMI && | |||
218 | (ExitMI->isCall() || ExitMI->isBarrier()); | |||
219 | if (ExitMI && AllDepKnown) { | |||
220 | // If it's a call or a barrier, add dependencies on the defs and uses of | |||
221 | // instruction. | |||
222 | for (unsigned i = 0, e = ExitMI->getNumOperands(); i != e; ++i) { | |||
223 | const MachineOperand &MO = ExitMI->getOperand(i); | |||
224 | if (!MO.isReg() || MO.isDef()) continue; | |||
225 | unsigned Reg = MO.getReg(); | |||
226 | if (Reg == 0) continue; | |||
227 | ||||
228 | if (TRI->isPhysicalRegister(Reg)) | |||
229 | Uses.insert(PhysRegSUOper(&ExitSU, -1, Reg)); | |||
230 | else if (MO.readsReg()) // ignore undef operands | |||
231 | addVRegUseDeps(&ExitSU, i); | |||
232 | } | |||
233 | } else { | |||
234 | // For others, e.g. fallthrough, conditional branch, assume the exit | |||
235 | // uses all the registers that are livein to the successor blocks. | |||
236 | assert(Uses.empty() && "Uses in set before adding deps?")((Uses.empty() && "Uses in set before adding deps?") ? static_cast<void> (0) : __assert_fail ("Uses.empty() && \"Uses in set before adding deps?\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 236, __PRETTY_FUNCTION__)); | |||
237 | for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(), | |||
238 | SE = BB->succ_end(); SI != SE; ++SI) | |||
239 | for (const auto &LI : (*SI)->liveins()) { | |||
240 | if (!Uses.contains(LI.PhysReg)) | |||
241 | Uses.insert(PhysRegSUOper(&ExitSU, -1, LI.PhysReg)); | |||
242 | } | |||
243 | } | |||
244 | } | |||
245 | ||||
246 | /// MO is an operand of SU's instruction that defines a physical register. Add | |||
247 | /// data dependencies from SU to any uses of the physical register. | |||
248 | void ScheduleDAGInstrs::addPhysRegDataDeps(SUnit *SU, unsigned OperIdx) { | |||
249 | const MachineOperand &MO = SU->getInstr()->getOperand(OperIdx); | |||
250 | assert(MO.isDef() && "expect physreg def")((MO.isDef() && "expect physreg def") ? static_cast< void> (0) : __assert_fail ("MO.isDef() && \"expect physreg def\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 250, __PRETTY_FUNCTION__)); | |||
251 | ||||
252 | // Ask the target if address-backscheduling is desirable, and if so how much. | |||
253 | const TargetSubtargetInfo &ST = MF.getSubtarget(); | |||
254 | ||||
255 | for (MCRegAliasIterator Alias(MO.getReg(), TRI, true); | |||
256 | Alias.isValid(); ++Alias) { | |||
257 | if (!Uses.contains(*Alias)) | |||
258 | continue; | |||
259 | for (Reg2SUnitsMap::iterator I = Uses.find(*Alias); I != Uses.end(); ++I) { | |||
260 | SUnit *UseSU = I->SU; | |||
261 | if (UseSU == SU) | |||
262 | continue; | |||
263 | ||||
264 | // Adjust the dependence latency using operand def/use information, | |||
265 | // then allow the target to perform its own adjustments. | |||
266 | int UseOp = I->OpIdx; | |||
267 | MachineInstr *RegUse = nullptr; | |||
268 | SDep Dep; | |||
269 | if (UseOp < 0) | |||
270 | Dep = SDep(SU, SDep::Artificial); | |||
271 | else { | |||
272 | // Set the hasPhysRegDefs only for physreg defs that have a use within | |||
273 | // the scheduling region. | |||
274 | SU->hasPhysRegDefs = true; | |||
275 | Dep = SDep(SU, SDep::Data, *Alias); | |||
276 | RegUse = UseSU->getInstr(); | |||
277 | } | |||
278 | Dep.setLatency( | |||
279 | SchedModel.computeOperandLatency(SU->getInstr(), OperIdx, RegUse, | |||
280 | UseOp)); | |||
281 | ||||
282 | ST.adjustSchedDependency(SU, UseSU, Dep); | |||
283 | UseSU->addPred(Dep); | |||
284 | } | |||
285 | } | |||
286 | } | |||
287 | ||||
288 | /// addPhysRegDeps - Add register dependencies (data, anti, and output) from | |||
289 | /// this SUnit to following instructions in the same scheduling region that | |||
290 | /// depend the physical register referenced at OperIdx. | |||
291 | void ScheduleDAGInstrs::addPhysRegDeps(SUnit *SU, unsigned OperIdx) { | |||
292 | MachineInstr *MI = SU->getInstr(); | |||
293 | MachineOperand &MO = MI->getOperand(OperIdx); | |||
294 | ||||
295 | // Optionally add output and anti dependencies. For anti | |||
296 | // dependencies we use a latency of 0 because for a multi-issue | |||
297 | // target we want to allow the defining instruction to issue | |||
298 | // in the same cycle as the using instruction. | |||
299 | // TODO: Using a latency of 1 here for output dependencies assumes | |||
300 | // there's no cost for reusing registers. | |||
301 | SDep::Kind Kind = MO.isUse() ? SDep::Anti : SDep::Output; | |||
302 | for (MCRegAliasIterator Alias(MO.getReg(), TRI, true); | |||
303 | Alias.isValid(); ++Alias) { | |||
304 | if (!Defs.contains(*Alias)) | |||
305 | continue; | |||
306 | for (Reg2SUnitsMap::iterator I = Defs.find(*Alias); I != Defs.end(); ++I) { | |||
307 | SUnit *DefSU = I->SU; | |||
308 | if (DefSU == &ExitSU) | |||
309 | continue; | |||
310 | if (DefSU != SU && | |||
311 | (Kind != SDep::Output || !MO.isDead() || | |||
312 | !DefSU->getInstr()->registerDefIsDead(*Alias))) { | |||
313 | if (Kind == SDep::Anti) | |||
314 | DefSU->addPred(SDep(SU, Kind, /*Reg=*/*Alias)); | |||
315 | else { | |||
316 | SDep Dep(SU, Kind, /*Reg=*/*Alias); | |||
317 | Dep.setLatency( | |||
318 | SchedModel.computeOutputLatency(MI, OperIdx, DefSU->getInstr())); | |||
319 | DefSU->addPred(Dep); | |||
320 | } | |||
321 | } | |||
322 | } | |||
323 | } | |||
324 | ||||
325 | if (!MO.isDef()) { | |||
326 | SU->hasPhysRegUses = true; | |||
327 | // Either insert a new Reg2SUnits entry with an empty SUnits list, or | |||
328 | // retrieve the existing SUnits list for this register's uses. | |||
329 | // Push this SUnit on the use list. | |||
330 | Uses.insert(PhysRegSUOper(SU, OperIdx, MO.getReg())); | |||
331 | if (RemoveKillFlags) | |||
332 | MO.setIsKill(false); | |||
333 | } | |||
334 | else { | |||
335 | addPhysRegDataDeps(SU, OperIdx); | |||
336 | unsigned Reg = MO.getReg(); | |||
337 | ||||
338 | // clear this register's use list | |||
339 | if (Uses.contains(Reg)) | |||
340 | Uses.eraseAll(Reg); | |||
341 | ||||
342 | if (!MO.isDead()) { | |||
343 | Defs.eraseAll(Reg); | |||
344 | } else if (SU->isCall) { | |||
345 | // Calls will not be reordered because of chain dependencies (see | |||
346 | // below). Since call operands are dead, calls may continue to be added | |||
347 | // to the DefList making dependence checking quadratic in the size of | |||
348 | // the block. Instead, we leave only one call at the back of the | |||
349 | // DefList. | |||
350 | Reg2SUnitsMap::RangePair P = Defs.equal_range(Reg); | |||
351 | Reg2SUnitsMap::iterator B = P.first; | |||
352 | Reg2SUnitsMap::iterator I = P.second; | |||
353 | for (bool isBegin = I == B; !isBegin; /* empty */) { | |||
354 | isBegin = (--I) == B; | |||
355 | if (!I->SU->isCall) | |||
356 | break; | |||
357 | I = Defs.erase(I); | |||
358 | } | |||
359 | } | |||
360 | ||||
361 | // Defs are pushed in the order they are visited and never reordered. | |||
362 | Defs.insert(PhysRegSUOper(SU, OperIdx, Reg)); | |||
363 | } | |||
364 | } | |||
365 | ||||
366 | /// addVRegDefDeps - Add register output and data dependencies from this SUnit | |||
367 | /// to instructions that occur later in the same scheduling region if they read | |||
368 | /// from or write to the virtual register defined at OperIdx. | |||
369 | /// | |||
370 | /// TODO: Hoist loop induction variable increments. This has to be | |||
371 | /// reevaluated. Generally, IV scheduling should be done before coalescing. | |||
372 | void ScheduleDAGInstrs::addVRegDefDeps(SUnit *SU, unsigned OperIdx) { | |||
373 | const MachineInstr *MI = SU->getInstr(); | |||
374 | unsigned Reg = MI->getOperand(OperIdx).getReg(); | |||
375 | ||||
376 | // Singly defined vregs do not have output/anti dependencies. | |||
377 | // The current operand is a def, so we have at least one. | |||
378 | // Check here if there are any others... | |||
379 | if (MRI.hasOneDef(Reg)) | |||
380 | return; | |||
381 | ||||
382 | // Add output dependence to the next nearest def of this vreg. | |||
383 | // | |||
384 | // Unless this definition is dead, the output dependence should be | |||
385 | // transitively redundant with antidependencies from this definition's | |||
386 | // uses. We're conservative for now until we have a way to guarantee the uses | |||
387 | // are not eliminated sometime during scheduling. The output dependence edge | |||
388 | // is also useful if output latency exceeds def-use latency. | |||
389 | VReg2SUnitMap::iterator DefI = VRegDefs.find(Reg); | |||
390 | if (DefI == VRegDefs.end()) | |||
391 | VRegDefs.insert(VReg2SUnit(Reg, SU)); | |||
392 | else { | |||
393 | SUnit *DefSU = DefI->SU; | |||
394 | if (DefSU != SU && DefSU != &ExitSU) { | |||
395 | SDep Dep(SU, SDep::Output, Reg); | |||
396 | Dep.setLatency( | |||
397 | SchedModel.computeOutputLatency(MI, OperIdx, DefSU->getInstr())); | |||
398 | DefSU->addPred(Dep); | |||
399 | } | |||
400 | DefI->SU = SU; | |||
401 | } | |||
402 | } | |||
403 | ||||
404 | /// addVRegUseDeps - Add a register data dependency if the instruction that | |||
405 | /// defines the virtual register used at OperIdx is mapped to an SUnit. Add a | |||
406 | /// register antidependency from this SUnit to instructions that occur later in | |||
407 | /// the same scheduling region if they write the virtual register. | |||
408 | /// | |||
409 | /// TODO: Handle ExitSU "uses" properly. | |||
410 | void ScheduleDAGInstrs::addVRegUseDeps(SUnit *SU, unsigned OperIdx) { | |||
411 | MachineInstr *MI = SU->getInstr(); | |||
412 | unsigned Reg = MI->getOperand(OperIdx).getReg(); | |||
413 | ||||
414 | // Record this local VReg use. | |||
415 | VReg2UseMap::iterator UI = VRegUses.find(Reg); | |||
416 | for (; UI != VRegUses.end(); ++UI) { | |||
417 | if (UI->SU == SU) | |||
418 | break; | |||
419 | } | |||
420 | if (UI == VRegUses.end()) | |||
421 | VRegUses.insert(VReg2SUnit(Reg, SU)); | |||
422 | ||||
423 | // Lookup this operand's reaching definition. | |||
424 | assert(LIS && "vreg dependencies requires LiveIntervals")((LIS && "vreg dependencies requires LiveIntervals") ? static_cast<void> (0) : __assert_fail ("LIS && \"vreg dependencies requires LiveIntervals\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 424, __PRETTY_FUNCTION__)); | |||
425 | LiveQueryResult LRQ | |||
426 | = LIS->getInterval(Reg).Query(LIS->getInstructionIndex(MI)); | |||
427 | VNInfo *VNI = LRQ.valueIn(); | |||
428 | ||||
429 | // VNI will be valid because MachineOperand::readsReg() is checked by caller. | |||
430 | assert(VNI && "No value to read by operand")((VNI && "No value to read by operand") ? static_cast <void> (0) : __assert_fail ("VNI && \"No value to read by operand\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 430, __PRETTY_FUNCTION__)); | |||
431 | MachineInstr *Def = LIS->getInstructionFromIndex(VNI->def); | |||
432 | // Phis and other noninstructions (after coalescing) have a NULL Def. | |||
433 | if (Def) { | |||
434 | SUnit *DefSU = getSUnit(Def); | |||
435 | if (DefSU) { | |||
436 | // The reaching Def lives within this scheduling region. | |||
437 | // Create a data dependence. | |||
438 | SDep dep(DefSU, SDep::Data, Reg); | |||
439 | // Adjust the dependence latency using operand def/use information, then | |||
440 | // allow the target to perform its own adjustments. | |||
441 | int DefOp = Def->findRegisterDefOperandIdx(Reg); | |||
442 | dep.setLatency(SchedModel.computeOperandLatency(Def, DefOp, MI, OperIdx)); | |||
443 | ||||
444 | const TargetSubtargetInfo &ST = MF.getSubtarget(); | |||
445 | ST.adjustSchedDependency(DefSU, SU, const_cast<SDep &>(dep)); | |||
446 | SU->addPred(dep); | |||
447 | } | |||
448 | } | |||
449 | ||||
450 | // Add antidependence to the following def of the vreg it uses. | |||
451 | VReg2SUnitMap::iterator DefI = VRegDefs.find(Reg); | |||
452 | if (DefI != VRegDefs.end() && DefI->SU != SU) | |||
453 | DefI->SU->addPred(SDep(SU, SDep::Anti, Reg)); | |||
454 | } | |||
455 | ||||
456 | /// Return true if MI is an instruction we are unable to reason about | |||
457 | /// (like a call or something with unmodeled side effects). | |||
458 | static inline bool isGlobalMemoryObject(AliasAnalysis *AA, MachineInstr *MI) { | |||
459 | return MI->isCall() || MI->hasUnmodeledSideEffects() || | |||
460 | (MI->hasOrderedMemoryRef() && | |||
461 | (!MI->mayLoad() || !MI->isInvariantLoad(AA))); | |||
462 | } | |||
463 | ||||
464 | // This MI might have either incomplete info, or known to be unsafe | |||
465 | // to deal with (i.e. volatile object). | |||
466 | static inline bool isUnsafeMemoryObject(MachineInstr *MI, | |||
467 | const MachineFrameInfo *MFI, | |||
468 | const DataLayout &DL) { | |||
469 | if (!MI || MI->memoperands_empty()) | |||
470 | return true; | |||
471 | // We purposefully do no check for hasOneMemOperand() here | |||
472 | // in hope to trigger an assert downstream in order to | |||
473 | // finish implementation. | |||
474 | if ((*MI->memoperands_begin())->isVolatile() || | |||
475 | MI->hasUnmodeledSideEffects()) | |||
476 | return true; | |||
477 | ||||
478 | if ((*MI->memoperands_begin())->getPseudoValue()) { | |||
479 | // Similarly to getUnderlyingObjectForInstr: | |||
480 | // For now, ignore PseudoSourceValues which may alias LLVM IR values | |||
481 | // because the code that uses this function has no way to cope with | |||
482 | // such aliases. | |||
483 | return true; | |||
484 | } | |||
485 | ||||
486 | const Value *V = (*MI->memoperands_begin())->getValue(); | |||
487 | if (!V) | |||
488 | return true; | |||
489 | ||||
490 | SmallVector<Value *, 4> Objs; | |||
491 | getUnderlyingObjects(V, Objs, DL); | |||
492 | for (Value *V : Objs) { | |||
493 | // Does this pointer refer to a distinct and identifiable object? | |||
494 | if (!isIdentifiedObject(V)) | |||
495 | return true; | |||
496 | } | |||
497 | ||||
498 | return false; | |||
499 | } | |||
500 | ||||
501 | /// This returns true if the two MIs need a chain edge between them. | |||
502 | /// If these are not even memory operations, we still may need | |||
503 | /// chain deps between them. The question really is - could | |||
504 | /// these two MIs be reordered during scheduling from memory dependency | |||
505 | /// point of view. | |||
506 | static bool MIsNeedChainEdge(AliasAnalysis *AA, const MachineFrameInfo *MFI, | |||
507 | const DataLayout &DL, MachineInstr *MIa, | |||
508 | MachineInstr *MIb) { | |||
509 | const MachineFunction *MF = MIa->getParent()->getParent(); | |||
510 | const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); | |||
511 | ||||
512 | // Cover a trivial case - no edge is need to itself. | |||
513 | if (MIa == MIb) | |||
514 | return false; | |||
515 | ||||
516 | // Let the target decide if memory accesses cannot possibly overlap. | |||
517 | if ((MIa->mayLoad() || MIa->mayStore()) && | |||
518 | (MIb->mayLoad() || MIb->mayStore())) | |||
519 | if (TII->areMemAccessesTriviallyDisjoint(MIa, MIb, AA)) | |||
520 | return false; | |||
521 | ||||
522 | // FIXME: Need to handle multiple memory operands to support all targets. | |||
523 | if (!MIa->hasOneMemOperand() || !MIb->hasOneMemOperand()) | |||
524 | return true; | |||
525 | ||||
526 | if (isUnsafeMemoryObject(MIa, MFI, DL) || isUnsafeMemoryObject(MIb, MFI, DL)) | |||
527 | return true; | |||
528 | ||||
529 | // If we are dealing with two "normal" loads, we do not need an edge | |||
530 | // between them - they could be reordered. | |||
531 | if (!MIa->mayStore() && !MIb->mayStore()) | |||
532 | return false; | |||
533 | ||||
534 | // To this point analysis is generic. From here on we do need AA. | |||
535 | if (!AA) | |||
536 | return true; | |||
537 | ||||
538 | MachineMemOperand *MMOa = *MIa->memoperands_begin(); | |||
539 | MachineMemOperand *MMOb = *MIb->memoperands_begin(); | |||
540 | ||||
541 | if (!MMOa->getValue() || !MMOb->getValue()) | |||
542 | return true; | |||
543 | ||||
544 | // The following interface to AA is fashioned after DAGCombiner::isAlias | |||
545 | // and operates with MachineMemOperand offset with some important | |||
546 | // assumptions: | |||
547 | // - LLVM fundamentally assumes flat address spaces. | |||
548 | // - MachineOperand offset can *only* result from legalization and | |||
549 | // cannot affect queries other than the trivial case of overlap | |||
550 | // checking. | |||
551 | // - These offsets never wrap and never step outside | |||
552 | // of allocated objects. | |||
553 | // - There should never be any negative offsets here. | |||
554 | // | |||
555 | // FIXME: Modify API to hide this math from "user" | |||
556 | // FIXME: Even before we go to AA we can reason locally about some | |||
557 | // memory objects. It can save compile time, and possibly catch some | |||
558 | // corner cases not currently covered. | |||
559 | ||||
560 | assert ((MMOa->getOffset() >= 0) && "Negative MachineMemOperand offset")(((MMOa->getOffset() >= 0) && "Negative MachineMemOperand offset" ) ? static_cast<void> (0) : __assert_fail ("(MMOa->getOffset() >= 0) && \"Negative MachineMemOperand offset\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 560, __PRETTY_FUNCTION__)); | |||
561 | assert ((MMOb->getOffset() >= 0) && "Negative MachineMemOperand offset")(((MMOb->getOffset() >= 0) && "Negative MachineMemOperand offset" ) ? static_cast<void> (0) : __assert_fail ("(MMOb->getOffset() >= 0) && \"Negative MachineMemOperand offset\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 561, __PRETTY_FUNCTION__)); | |||
562 | ||||
563 | int64_t MinOffset = std::min(MMOa->getOffset(), MMOb->getOffset()); | |||
564 | int64_t Overlapa = MMOa->getSize() + MMOa->getOffset() - MinOffset; | |||
565 | int64_t Overlapb = MMOb->getSize() + MMOb->getOffset() - MinOffset; | |||
566 | ||||
567 | AliasResult AAResult = | |||
568 | AA->alias(MemoryLocation(MMOa->getValue(), Overlapa, | |||
569 | UseTBAA ? MMOa->getAAInfo() : AAMDNodes()), | |||
570 | MemoryLocation(MMOb->getValue(), Overlapb, | |||
571 | UseTBAA ? MMOb->getAAInfo() : AAMDNodes())); | |||
572 | ||||
573 | return (AAResult != NoAlias); | |||
574 | } | |||
575 | ||||
576 | /// This recursive function iterates over chain deps of SUb looking for | |||
577 | /// "latest" node that needs a chain edge to SUa. | |||
578 | static unsigned iterateChainSucc(AliasAnalysis *AA, const MachineFrameInfo *MFI, | |||
579 | const DataLayout &DL, SUnit *SUa, SUnit *SUb, | |||
580 | SUnit *ExitSU, unsigned *Depth, | |||
581 | SmallPtrSetImpl<const SUnit *> &Visited) { | |||
582 | if (!SUa || !SUb || SUb == ExitSU) | |||
583 | return *Depth; | |||
584 | ||||
585 | // Remember visited nodes. | |||
586 | if (!Visited.insert(SUb).second) | |||
587 | return *Depth; | |||
588 | // If there is _some_ dependency already in place, do not | |||
589 | // descend any further. | |||
590 | // TODO: Need to make sure that if that dependency got eliminated or ignored | |||
591 | // for any reason in the future, we would not violate DAG topology. | |||
592 | // Currently it does not happen, but makes an implicit assumption about | |||
593 | // future implementation. | |||
594 | // | |||
595 | // Independently, if we encounter node that is some sort of global | |||
596 | // object (like a call) we already have full set of dependencies to it | |||
597 | // and we can stop descending. | |||
598 | if (SUa->isSucc(SUb) || | |||
599 | isGlobalMemoryObject(AA, SUb->getInstr())) | |||
600 | return *Depth; | |||
601 | ||||
602 | // If we do need an edge, or we have exceeded depth budget, | |||
603 | // add that edge to the predecessors chain of SUb, | |||
604 | // and stop descending. | |||
605 | if (*Depth > 200 || | |||
606 | MIsNeedChainEdge(AA, MFI, DL, SUa->getInstr(), SUb->getInstr())) { | |||
607 | SUb->addPred(SDep(SUa, SDep::MayAliasMem)); | |||
608 | return *Depth; | |||
609 | } | |||
610 | // Track current depth. | |||
611 | (*Depth)++; | |||
612 | // Iterate over memory dependencies only. | |||
613 | for (SUnit::const_succ_iterator I = SUb->Succs.begin(), E = SUb->Succs.end(); | |||
614 | I != E; ++I) | |||
615 | if (I->isNormalMemoryOrBarrier()) | |||
616 | iterateChainSucc(AA, MFI, DL, SUa, I->getSUnit(), ExitSU, Depth, Visited); | |||
617 | return *Depth; | |||
618 | } | |||
619 | ||||
620 | /// This function assumes that "downward" from SU there exist | |||
621 | /// tail/leaf of already constructed DAG. It iterates downward and | |||
622 | /// checks whether SU can be aliasing any node dominated | |||
623 | /// by it. | |||
624 | static void adjustChainDeps(AliasAnalysis *AA, const MachineFrameInfo *MFI, | |||
625 | const DataLayout &DL, SUnit *SU, SUnit *ExitSU, | |||
626 | std::set<SUnit *> &CheckList, | |||
627 | unsigned LatencyToLoad) { | |||
628 | if (!SU) | |||
629 | return; | |||
630 | ||||
631 | SmallPtrSet<const SUnit*, 16> Visited; | |||
632 | unsigned Depth = 0; | |||
633 | ||||
634 | for (std::set<SUnit *>::iterator I = CheckList.begin(), IE = CheckList.end(); | |||
635 | I != IE; ++I) { | |||
636 | if (SU == *I) | |||
637 | continue; | |||
638 | if (MIsNeedChainEdge(AA, MFI, DL, SU->getInstr(), (*I)->getInstr())) { | |||
639 | SDep Dep(SU, SDep::MayAliasMem); | |||
640 | Dep.setLatency(((*I)->getInstr()->mayLoad()) ? LatencyToLoad : 0); | |||
641 | (*I)->addPred(Dep); | |||
642 | } | |||
643 | ||||
644 | // Iterate recursively over all previously added memory chain | |||
645 | // successors. Keep track of visited nodes. | |||
646 | for (SUnit::const_succ_iterator J = (*I)->Succs.begin(), | |||
647 | JE = (*I)->Succs.end(); J != JE; ++J) | |||
648 | if (J->isNormalMemoryOrBarrier()) | |||
649 | iterateChainSucc(AA, MFI, DL, SU, J->getSUnit(), ExitSU, &Depth, | |||
650 | Visited); | |||
651 | } | |||
652 | } | |||
653 | ||||
654 | /// Check whether two objects need a chain edge, if so, add it | |||
655 | /// otherwise remember the rejected SU. | |||
656 | static inline void addChainDependency(AliasAnalysis *AA, | |||
657 | const MachineFrameInfo *MFI, | |||
658 | const DataLayout &DL, SUnit *SUa, | |||
659 | SUnit *SUb, std::set<SUnit *> &RejectList, | |||
660 | unsigned TrueMemOrderLatency = 0, | |||
661 | bool isNormalMemory = false) { | |||
662 | // If this is a false dependency, | |||
663 | // do not add the edge, but remember the rejected node. | |||
664 | if (MIsNeedChainEdge(AA, MFI, DL, SUa->getInstr(), SUb->getInstr())) { | |||
665 | SDep Dep(SUa, isNormalMemory ? SDep::MayAliasMem : SDep::Barrier); | |||
666 | Dep.setLatency(TrueMemOrderLatency); | |||
667 | SUb->addPred(Dep); | |||
668 | } | |||
669 | else { | |||
670 | // Duplicate entries should be ignored. | |||
671 | RejectList.insert(SUb); | |||
672 | DEBUG(dbgs() << "\tReject chain dep between SU("do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { dbgs() << "\tReject chain dep between SU(" << SUa->NodeNum << ") and SU(" << SUb-> NodeNum << ")\n"; } } while (0) | |||
673 | << SUa->NodeNum << ") and SU("do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { dbgs() << "\tReject chain dep between SU(" << SUa->NodeNum << ") and SU(" << SUb-> NodeNum << ")\n"; } } while (0) | |||
674 | << SUb->NodeNum << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { dbgs() << "\tReject chain dep between SU(" << SUa->NodeNum << ") and SU(" << SUb-> NodeNum << ")\n"; } } while (0); | |||
675 | } | |||
676 | } | |||
677 | ||||
678 | /// Create an SUnit for each real instruction, numbered in top-down topological | |||
679 | /// order. The instruction order A < B, implies that no edge exists from B to A. | |||
680 | /// | |||
681 | /// Map each real instruction to its SUnit. | |||
682 | /// | |||
683 | /// After initSUnits, the SUnits vector cannot be resized and the scheduler may | |||
684 | /// hang onto SUnit pointers. We may relax this in the future by using SUnit IDs | |||
685 | /// instead of pointers. | |||
686 | /// | |||
687 | /// MachineScheduler relies on initSUnits numbering the nodes by their order in | |||
688 | /// the original instruction list. | |||
689 | void ScheduleDAGInstrs::initSUnits() { | |||
690 | // We'll be allocating one SUnit for each real instruction in the region, | |||
691 | // which is contained within a basic block. | |||
692 | SUnits.reserve(NumRegionInstrs); | |||
693 | ||||
694 | for (MachineBasicBlock::iterator I = RegionBegin; I != RegionEnd; ++I) { | |||
695 | MachineInstr *MI = I; | |||
696 | if (MI->isDebugValue()) | |||
697 | continue; | |||
698 | ||||
699 | SUnit *SU = newSUnit(MI); | |||
700 | MISUnitMap[MI] = SU; | |||
701 | ||||
702 | SU->isCall = MI->isCall(); | |||
703 | SU->isCommutable = MI->isCommutable(); | |||
704 | ||||
705 | // Assign the Latency field of SU using target-provided information. | |||
706 | SU->Latency = SchedModel.computeInstrLatency(SU->getInstr()); | |||
707 | ||||
708 | // If this SUnit uses a reserved or unbuffered resource, mark it as such. | |||
709 | // | |||
710 | // Reserved resources block an instruction from issuing and stall the | |||
711 | // entire pipeline. These are identified by BufferSize=0. | |||
712 | // | |||
713 | // Unbuffered resources prevent execution of subsequent instructions that | |||
714 | // require the same resources. This is used for in-order execution pipelines | |||
715 | // within an out-of-order core. These are identified by BufferSize=1. | |||
716 | if (SchedModel.hasInstrSchedModel()) { | |||
717 | const MCSchedClassDesc *SC = getSchedClass(SU); | |||
718 | for (TargetSchedModel::ProcResIter | |||
719 | PI = SchedModel.getWriteProcResBegin(SC), | |||
720 | PE = SchedModel.getWriteProcResEnd(SC); PI != PE; ++PI) { | |||
721 | switch (SchedModel.getProcResource(PI->ProcResourceIdx)->BufferSize) { | |||
722 | case 0: | |||
723 | SU->hasReservedResource = true; | |||
724 | break; | |||
725 | case 1: | |||
726 | SU->isUnbuffered = true; | |||
727 | break; | |||
728 | default: | |||
729 | break; | |||
730 | } | |||
731 | } | |||
732 | } | |||
733 | } | |||
734 | } | |||
735 | ||||
736 | /// If RegPressure is non-null, compute register pressure as a side effect. The | |||
737 | /// DAG builder is an efficient place to do it because it already visits | |||
738 | /// operands. | |||
739 | void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, | |||
740 | RegPressureTracker *RPTracker, | |||
741 | PressureDiffs *PDiffs) { | |||
742 | const TargetSubtargetInfo &ST = MF.getSubtarget(); | |||
743 | bool UseAA = EnableAASchedMI.getNumOccurrences() > 0 ? EnableAASchedMI | |||
| ||||
744 | : ST.useAA(); | |||
745 | AliasAnalysis *AAForDep = UseAA ? AA : nullptr; | |||
746 | ||||
747 | MISUnitMap.clear(); | |||
748 | ScheduleDAG::clearDAG(); | |||
749 | ||||
750 | // Create an SUnit for each real instruction. | |||
751 | initSUnits(); | |||
752 | ||||
753 | if (PDiffs) | |||
754 | PDiffs->init(SUnits.size()); | |||
755 | ||||
756 | // We build scheduling units by walking a block's instruction list from bottom | |||
757 | // to top. | |||
758 | ||||
759 | // Remember where a generic side-effecting instruction is as we proceed. | |||
760 | SUnit *BarrierChain = nullptr, *AliasChain = nullptr; | |||
761 | ||||
762 | // Memory references to specific known memory locations are tracked | |||
763 | // so that they can be given more precise dependencies. We track | |||
764 | // separately the known memory locations that may alias and those | |||
765 | // that are known not to alias | |||
766 | MapVector<ValueType, std::vector<SUnit *> > AliasMemDefs, NonAliasMemDefs; | |||
767 | MapVector<ValueType, std::vector<SUnit *> > AliasMemUses, NonAliasMemUses; | |||
768 | std::set<SUnit*> RejectMemNodes; | |||
769 | ||||
770 | // Remove any stale debug info; sometimes BuildSchedGraph is called again | |||
771 | // without emitting the info from the previous call. | |||
772 | DbgValues.clear(); | |||
773 | FirstDbgValue = nullptr; | |||
774 | ||||
775 | assert(Defs.empty() && Uses.empty() &&((Defs.empty() && Uses.empty() && "Only BuildGraph should update Defs/Uses" ) ? static_cast<void> (0) : __assert_fail ("Defs.empty() && Uses.empty() && \"Only BuildGraph should update Defs/Uses\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 776, __PRETTY_FUNCTION__)) | |||
776 | "Only BuildGraph should update Defs/Uses")((Defs.empty() && Uses.empty() && "Only BuildGraph should update Defs/Uses" ) ? static_cast<void> (0) : __assert_fail ("Defs.empty() && Uses.empty() && \"Only BuildGraph should update Defs/Uses\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 776, __PRETTY_FUNCTION__)); | |||
777 | Defs.setUniverse(TRI->getNumRegs()); | |||
778 | Uses.setUniverse(TRI->getNumRegs()); | |||
779 | ||||
780 | assert(VRegDefs.empty() && "Only BuildSchedGraph may access VRegDefs")((VRegDefs.empty() && "Only BuildSchedGraph may access VRegDefs" ) ? static_cast<void> (0) : __assert_fail ("VRegDefs.empty() && \"Only BuildSchedGraph may access VRegDefs\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 780, __PRETTY_FUNCTION__)); | |||
781 | VRegUses.clear(); | |||
782 | VRegDefs.setUniverse(MRI.getNumVirtRegs()); | |||
783 | VRegUses.setUniverse(MRI.getNumVirtRegs()); | |||
784 | ||||
785 | // Model data dependencies between instructions being scheduled and the | |||
786 | // ExitSU. | |||
787 | addSchedBarrierDeps(); | |||
788 | ||||
789 | // Walk the list of instructions, from bottom moving up. | |||
790 | MachineInstr *DbgMI = nullptr; | |||
791 | for (MachineBasicBlock::iterator MII = RegionEnd, MIE = RegionBegin; | |||
792 | MII != MIE; --MII) { | |||
793 | MachineInstr *MI = std::prev(MII); | |||
794 | if (MI && DbgMI) { | |||
795 | DbgValues.push_back(std::make_pair(DbgMI, MI)); | |||
796 | DbgMI = nullptr; | |||
797 | } | |||
798 | ||||
799 | if (MI->isDebugValue()) { | |||
| ||||
800 | DbgMI = MI; | |||
801 | continue; | |||
802 | } | |||
803 | SUnit *SU = MISUnitMap[MI]; | |||
804 | assert(SU && "No SUnit mapped to this MI")((SU && "No SUnit mapped to this MI") ? static_cast< void> (0) : __assert_fail ("SU && \"No SUnit mapped to this MI\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 804, __PRETTY_FUNCTION__)); | |||
805 | ||||
806 | if (RPTracker) { | |||
807 | PressureDiff *PDiff = PDiffs ? &(*PDiffs)[SU->NodeNum] : nullptr; | |||
808 | RPTracker->recede(/*LiveUses=*/nullptr, PDiff); | |||
809 | assert(RPTracker->getPos() == std::prev(MII) &&((RPTracker->getPos() == std::prev(MII) && "RPTracker can't find MI" ) ? static_cast<void> (0) : __assert_fail ("RPTracker->getPos() == std::prev(MII) && \"RPTracker can't find MI\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 810, __PRETTY_FUNCTION__)) | |||
810 | "RPTracker can't find MI")((RPTracker->getPos() == std::prev(MII) && "RPTracker can't find MI" ) ? static_cast<void> (0) : __assert_fail ("RPTracker->getPos() == std::prev(MII) && \"RPTracker can't find MI\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 810, __PRETTY_FUNCTION__)); | |||
811 | } | |||
812 | ||||
813 | assert((((CanHandleTerminators || (!MI->isTerminator() && !MI->isPosition())) && "Cannot schedule terminators or labels!" ) ? static_cast<void> (0) : __assert_fail ("(CanHandleTerminators || (!MI->isTerminator() && !MI->isPosition())) && \"Cannot schedule terminators or labels!\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 815, __PRETTY_FUNCTION__)) | |||
814 | (CanHandleTerminators || (!MI->isTerminator() && !MI->isPosition())) &&(((CanHandleTerminators || (!MI->isTerminator() && !MI->isPosition())) && "Cannot schedule terminators or labels!" ) ? static_cast<void> (0) : __assert_fail ("(CanHandleTerminators || (!MI->isTerminator() && !MI->isPosition())) && \"Cannot schedule terminators or labels!\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 815, __PRETTY_FUNCTION__)) | |||
815 | "Cannot schedule terminators or labels!")(((CanHandleTerminators || (!MI->isTerminator() && !MI->isPosition())) && "Cannot schedule terminators or labels!" ) ? static_cast<void> (0) : __assert_fail ("(CanHandleTerminators || (!MI->isTerminator() && !MI->isPosition())) && \"Cannot schedule terminators or labels!\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 815, __PRETTY_FUNCTION__)); | |||
816 | ||||
817 | // Add register-based dependencies (data, anti, and output). | |||
818 | bool HasVRegDef = false; | |||
819 | for (unsigned j = 0, n = MI->getNumOperands(); j != n; ++j) { | |||
820 | const MachineOperand &MO = MI->getOperand(j); | |||
821 | if (!MO.isReg()) continue; | |||
822 | unsigned Reg = MO.getReg(); | |||
823 | if (Reg == 0) continue; | |||
824 | ||||
825 | if (TRI->isPhysicalRegister(Reg)) | |||
826 | addPhysRegDeps(SU, j); | |||
827 | else { | |||
828 | if (MO.isDef()) { | |||
829 | HasVRegDef = true; | |||
830 | addVRegDefDeps(SU, j); | |||
831 | } | |||
832 | else if (MO.readsReg()) // ignore undef operands | |||
833 | addVRegUseDeps(SU, j); | |||
834 | } | |||
835 | } | |||
836 | // If we haven't seen any uses in this scheduling region, create a | |||
837 | // dependence edge to ExitSU to model the live-out latency. This is required | |||
838 | // for vreg defs with no in-region use, and prefetches with no vreg def. | |||
839 | // | |||
840 | // FIXME: NumDataSuccs would be more precise than NumSuccs here. This | |||
841 | // check currently relies on being called before adding chain deps. | |||
842 | if (SU->NumSuccs == 0 && SU->Latency > 1 | |||
843 | && (HasVRegDef || MI->mayLoad())) { | |||
844 | SDep Dep(SU, SDep::Artificial); | |||
845 | Dep.setLatency(SU->Latency - 1); | |||
846 | ExitSU.addPred(Dep); | |||
847 | } | |||
848 | ||||
849 | // Add chain dependencies. | |||
850 | // Chain dependencies used to enforce memory order should have | |||
851 | // latency of 0 (except for true dependency of Store followed by | |||
852 | // aliased Load... we estimate that with a single cycle of latency | |||
853 | // assuming the hardware will bypass) | |||
854 | // Note that isStoreToStackSlot and isLoadFromStackSLot are not usable | |||
855 | // after stack slots are lowered to actual addresses. | |||
856 | // TODO: Use an AliasAnalysis and do real alias-analysis queries, and | |||
857 | // produce more precise dependence information. | |||
858 | unsigned TrueMemOrderLatency = MI->mayStore() ? 1 : 0; | |||
859 | if (isGlobalMemoryObject(AA, MI)) { | |||
860 | // Be conservative with these and add dependencies on all memory | |||
861 | // references, even those that are known to not alias. | |||
862 | for (MapVector<ValueType, std::vector<SUnit *> >::iterator I = | |||
863 | NonAliasMemDefs.begin(), E = NonAliasMemDefs.end(); I != E; ++I) { | |||
864 | for (unsigned i = 0, e = I->second.size(); i != e; ++i) { | |||
865 | I->second[i]->addPred(SDep(SU, SDep::Barrier)); | |||
866 | } | |||
867 | } | |||
868 | for (MapVector<ValueType, std::vector<SUnit *> >::iterator I = | |||
869 | NonAliasMemUses.begin(), E = NonAliasMemUses.end(); I != E; ++I) { | |||
870 | for (unsigned i = 0, e = I->second.size(); i != e; ++i) { | |||
871 | SDep Dep(SU, SDep::Barrier); | |||
872 | Dep.setLatency(TrueMemOrderLatency); | |||
873 | I->second[i]->addPred(Dep); | |||
874 | } | |||
875 | } | |||
876 | // Add SU to the barrier chain. | |||
877 | if (BarrierChain) | |||
878 | BarrierChain->addPred(SDep(SU, SDep::Barrier)); | |||
879 | BarrierChain = SU; | |||
880 | // This is a barrier event that acts as a pivotal node in the DAG, | |||
881 | // so it is safe to clear list of exposed nodes. | |||
882 | adjustChainDeps(AA, MFI, MF.getDataLayout(), SU, &ExitSU, RejectMemNodes, | |||
883 | TrueMemOrderLatency); | |||
884 | RejectMemNodes.clear(); | |||
885 | NonAliasMemDefs.clear(); | |||
886 | NonAliasMemUses.clear(); | |||
887 | ||||
888 | // fall-through | |||
889 | new_alias_chain: | |||
890 | // Chain all possibly aliasing memory references through SU. | |||
891 | if (AliasChain) { | |||
892 | unsigned ChainLatency = 0; | |||
893 | if (AliasChain->getInstr()->mayLoad()) | |||
894 | ChainLatency = TrueMemOrderLatency; | |||
895 | addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, AliasChain, | |||
896 | RejectMemNodes, ChainLatency); | |||
897 | } | |||
898 | AliasChain = SU; | |||
899 | for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k) | |||
900 | addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, | |||
901 | PendingLoads[k], RejectMemNodes, | |||
902 | TrueMemOrderLatency); | |||
903 | for (MapVector<ValueType, std::vector<SUnit *> >::iterator I = | |||
904 | AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I) { | |||
905 | for (unsigned i = 0, e = I->second.size(); i != e; ++i) | |||
906 | addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, | |||
907 | I->second[i], RejectMemNodes); | |||
908 | } | |||
909 | for (MapVector<ValueType, std::vector<SUnit *> >::iterator I = | |||
910 | AliasMemUses.begin(), E = AliasMemUses.end(); I != E; ++I) { | |||
911 | for (unsigned i = 0, e = I->second.size(); i != e; ++i) | |||
912 | addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, | |||
913 | I->second[i], RejectMemNodes, TrueMemOrderLatency); | |||
914 | } | |||
915 | adjustChainDeps(AA, MFI, MF.getDataLayout(), SU, &ExitSU, RejectMemNodes, | |||
916 | TrueMemOrderLatency); | |||
917 | PendingLoads.clear(); | |||
918 | AliasMemDefs.clear(); | |||
919 | AliasMemUses.clear(); | |||
920 | } else if (MI->mayStore()) { | |||
921 | // Add dependence on barrier chain, if needed. | |||
922 | // There is no point to check aliasing on barrier event. Even if | |||
923 | // SU and barrier _could_ be reordered, they should not. In addition, | |||
924 | // we have lost all RejectMemNodes below barrier. | |||
925 | if (BarrierChain) | |||
926 | BarrierChain->addPred(SDep(SU, SDep::Barrier)); | |||
927 | ||||
928 | UnderlyingObjectsVector Objs; | |||
929 | getUnderlyingObjectsForInstr(MI, MFI, Objs, MF.getDataLayout()); | |||
930 | ||||
931 | if (Objs.empty()) { | |||
932 | // Treat all other stores conservatively. | |||
933 | goto new_alias_chain; | |||
934 | } | |||
935 | ||||
936 | bool MayAlias = false; | |||
937 | for (UnderlyingObjectsVector::iterator K = Objs.begin(), KE = Objs.end(); | |||
938 | K != KE; ++K) { | |||
939 | ValueType V = K->getPointer(); | |||
940 | bool ThisMayAlias = K->getInt(); | |||
941 | if (ThisMayAlias) | |||
942 | MayAlias = true; | |||
943 | ||||
944 | // A store to a specific PseudoSourceValue. Add precise dependencies. | |||
945 | // Record the def in MemDefs, first adding a dep if there is | |||
946 | // an existing def. | |||
947 | MapVector<ValueType, std::vector<SUnit *> >::iterator I = | |||
948 | ((ThisMayAlias) ? AliasMemDefs.find(V) : NonAliasMemDefs.find(V)); | |||
949 | MapVector<ValueType, std::vector<SUnit *> >::iterator IE = | |||
950 | ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end()); | |||
951 | if (I != IE) { | |||
952 | for (unsigned i = 0, e = I->second.size(); i != e; ++i) | |||
953 | addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, | |||
954 | I->second[i], RejectMemNodes, 0, true); | |||
955 | ||||
956 | // If we're not using AA, then we only need one store per object. | |||
957 | if (!AAForDep) | |||
958 | I->second.clear(); | |||
959 | I->second.push_back(SU); | |||
960 | } else { | |||
961 | if (ThisMayAlias) { | |||
962 | if (!AAForDep) | |||
963 | AliasMemDefs[V].clear(); | |||
964 | AliasMemDefs[V].push_back(SU); | |||
965 | } else { | |||
966 | if (!AAForDep) | |||
967 | NonAliasMemDefs[V].clear(); | |||
968 | NonAliasMemDefs[V].push_back(SU); | |||
969 | } | |||
970 | } | |||
971 | // Handle the uses in MemUses, if there are any. | |||
972 | MapVector<ValueType, std::vector<SUnit *> >::iterator J = | |||
973 | ((ThisMayAlias) ? AliasMemUses.find(V) : NonAliasMemUses.find(V)); | |||
974 | MapVector<ValueType, std::vector<SUnit *> >::iterator JE = | |||
975 | ((ThisMayAlias) ? AliasMemUses.end() : NonAliasMemUses.end()); | |||
976 | if (J != JE) { | |||
977 | for (unsigned i = 0, e = J->second.size(); i != e; ++i) | |||
978 | addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, | |||
979 | J->second[i], RejectMemNodes, | |||
980 | TrueMemOrderLatency, true); | |||
981 | J->second.clear(); | |||
982 | } | |||
983 | } | |||
984 | if (MayAlias) { | |||
985 | // Add dependencies from all the PendingLoads, i.e. loads | |||
986 | // with no underlying object. | |||
987 | for (unsigned k = 0, m = PendingLoads.size(); k != m; ++k) | |||
988 | addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, | |||
989 | PendingLoads[k], RejectMemNodes, | |||
990 | TrueMemOrderLatency); | |||
991 | // Add dependence on alias chain, if needed. | |||
992 | if (AliasChain) | |||
993 | addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, AliasChain, | |||
994 | RejectMemNodes); | |||
995 | } | |||
996 | adjustChainDeps(AA, MFI, MF.getDataLayout(), SU, &ExitSU, RejectMemNodes, | |||
997 | TrueMemOrderLatency); | |||
998 | } else if (MI->mayLoad()) { | |||
999 | bool MayAlias = true; | |||
1000 | if (MI->isInvariantLoad(AA)) { | |||
1001 | // Invariant load, no chain dependencies needed! | |||
1002 | } else { | |||
1003 | UnderlyingObjectsVector Objs; | |||
1004 | getUnderlyingObjectsForInstr(MI, MFI, Objs, MF.getDataLayout()); | |||
1005 | ||||
1006 | if (Objs.empty()) { | |||
1007 | // A load with no underlying object. Depend on all | |||
1008 | // potentially aliasing stores. | |||
1009 | for (MapVector<ValueType, std::vector<SUnit *> >::iterator I = | |||
1010 | AliasMemDefs.begin(), E = AliasMemDefs.end(); I != E; ++I) | |||
1011 | for (unsigned i = 0, e = I->second.size(); i != e; ++i) | |||
1012 | addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, | |||
1013 | I->second[i], RejectMemNodes); | |||
1014 | ||||
1015 | PendingLoads.push_back(SU); | |||
1016 | MayAlias = true; | |||
1017 | } else { | |||
1018 | MayAlias = false; | |||
1019 | } | |||
1020 | ||||
1021 | for (UnderlyingObjectsVector::iterator | |||
1022 | J = Objs.begin(), JE = Objs.end(); J != JE; ++J) { | |||
1023 | ValueType V = J->getPointer(); | |||
1024 | bool ThisMayAlias = J->getInt(); | |||
1025 | ||||
1026 | if (ThisMayAlias) | |||
1027 | MayAlias = true; | |||
1028 | ||||
1029 | // A load from a specific PseudoSourceValue. Add precise dependencies. | |||
1030 | MapVector<ValueType, std::vector<SUnit *> >::iterator I = | |||
1031 | ((ThisMayAlias) ? AliasMemDefs.find(V) : NonAliasMemDefs.find(V)); | |||
1032 | MapVector<ValueType, std::vector<SUnit *> >::iterator IE = | |||
1033 | ((ThisMayAlias) ? AliasMemDefs.end() : NonAliasMemDefs.end()); | |||
1034 | if (I != IE) | |||
1035 | for (unsigned i = 0, e = I->second.size(); i != e; ++i) | |||
1036 | addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, | |||
1037 | I->second[i], RejectMemNodes, 0, true); | |||
1038 | if (ThisMayAlias) | |||
1039 | AliasMemUses[V].push_back(SU); | |||
1040 | else | |||
1041 | NonAliasMemUses[V].push_back(SU); | |||
1042 | } | |||
1043 | if (MayAlias) | |||
1044 | adjustChainDeps(AA, MFI, MF.getDataLayout(), SU, &ExitSU, | |||
1045 | RejectMemNodes, /*Latency=*/0); | |||
1046 | // Add dependencies on alias and barrier chains, if needed. | |||
1047 | if (MayAlias && AliasChain) | |||
1048 | addChainDependency(AAForDep, MFI, MF.getDataLayout(), SU, AliasChain, | |||
1049 | RejectMemNodes); | |||
1050 | if (BarrierChain) | |||
1051 | BarrierChain->addPred(SDep(SU, SDep::Barrier)); | |||
1052 | } | |||
1053 | } | |||
1054 | } | |||
1055 | if (DbgMI) | |||
1056 | FirstDbgValue = DbgMI; | |||
1057 | ||||
1058 | Defs.clear(); | |||
1059 | Uses.clear(); | |||
1060 | VRegDefs.clear(); | |||
1061 | PendingLoads.clear(); | |||
1062 | } | |||
1063 | ||||
1064 | /// \brief Initialize register live-range state for updating kills. | |||
1065 | void ScheduleDAGInstrs::startBlockForKills(MachineBasicBlock *BB) { | |||
1066 | // Start with no live registers. | |||
1067 | LiveRegs.reset(); | |||
1068 | ||||
1069 | // Examine the live-in regs of all successors. | |||
1070 | for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(), | |||
1071 | SE = BB->succ_end(); SI != SE; ++SI) { | |||
1072 | for (const auto &LI : (*SI)->liveins()) { | |||
1073 | // Repeat, for reg and all subregs. | |||
1074 | for (MCSubRegIterator SubRegs(LI.PhysReg, TRI, /*IncludeSelf=*/true); | |||
1075 | SubRegs.isValid(); ++SubRegs) | |||
1076 | LiveRegs.set(*SubRegs); | |||
1077 | } | |||
1078 | } | |||
1079 | } | |||
1080 | ||||
1081 | /// \brief If we change a kill flag on the bundle instruction implicit register | |||
1082 | /// operands, then we also need to propagate that to any instructions inside | |||
1083 | /// the bundle which had the same kill state. | |||
1084 | static void toggleBundleKillFlag(MachineInstr *MI, unsigned Reg, | |||
1085 | bool NewKillState) { | |||
1086 | if (MI->getOpcode() != TargetOpcode::BUNDLE) | |||
1087 | return; | |||
1088 | ||||
1089 | // Walk backwards from the last instruction in the bundle to the first. | |||
1090 | // Once we set a kill flag on an instruction, we bail out, as otherwise we | |||
1091 | // might set it on too many operands. We will clear as many flags as we | |||
1092 | // can though. | |||
1093 | MachineBasicBlock::instr_iterator Begin = MI->getIterator(); | |||
1094 | MachineBasicBlock::instr_iterator End = getBundleEnd(MI); | |||
1095 | while (Begin != End) { | |||
1096 | for (MachineOperand &MO : (--End)->operands()) { | |||
1097 | if (!MO.isReg() || MO.isDef() || Reg != MO.getReg()) | |||
1098 | continue; | |||
1099 | ||||
1100 | // DEBUG_VALUE nodes do not contribute to code generation and should | |||
1101 | // always be ignored. Failure to do so may result in trying to modify | |||
1102 | // KILL flags on DEBUG_VALUE nodes, which is distressing. | |||
1103 | if (MO.isDebug()) | |||
1104 | continue; | |||
1105 | ||||
1106 | // If the register has the internal flag then it could be killing an | |||
1107 | // internal def of the register. In this case, just skip. We only want | |||
1108 | // to toggle the flag on operands visible outside the bundle. | |||
1109 | if (MO.isInternalRead()) | |||
1110 | continue; | |||
1111 | ||||
1112 | if (MO.isKill() == NewKillState) | |||
1113 | continue; | |||
1114 | MO.setIsKill(NewKillState); | |||
1115 | if (NewKillState) | |||
1116 | return; | |||
1117 | } | |||
1118 | } | |||
1119 | } | |||
1120 | ||||
1121 | bool ScheduleDAGInstrs::toggleKillFlag(MachineInstr *MI, MachineOperand &MO) { | |||
1122 | // Setting kill flag... | |||
1123 | if (!MO.isKill()) { | |||
1124 | MO.setIsKill(true); | |||
1125 | toggleBundleKillFlag(MI, MO.getReg(), true); | |||
1126 | return false; | |||
1127 | } | |||
1128 | ||||
1129 | // If MO itself is live, clear the kill flag... | |||
1130 | if (LiveRegs.test(MO.getReg())) { | |||
1131 | MO.setIsKill(false); | |||
1132 | toggleBundleKillFlag(MI, MO.getReg(), false); | |||
1133 | return false; | |||
1134 | } | |||
1135 | ||||
1136 | // If any subreg of MO is live, then create an imp-def for that | |||
1137 | // subreg and keep MO marked as killed. | |||
1138 | MO.setIsKill(false); | |||
1139 | toggleBundleKillFlag(MI, MO.getReg(), false); | |||
1140 | bool AllDead = true; | |||
1141 | const unsigned SuperReg = MO.getReg(); | |||
1142 | MachineInstrBuilder MIB(MF, MI); | |||
1143 | for (MCSubRegIterator SubRegs(SuperReg, TRI); SubRegs.isValid(); ++SubRegs) { | |||
1144 | if (LiveRegs.test(*SubRegs)) { | |||
1145 | MIB.addReg(*SubRegs, RegState::ImplicitDefine); | |||
1146 | AllDead = false; | |||
1147 | } | |||
1148 | } | |||
1149 | ||||
1150 | if(AllDead) { | |||
1151 | MO.setIsKill(true); | |||
1152 | toggleBundleKillFlag(MI, MO.getReg(), true); | |||
1153 | } | |||
1154 | return false; | |||
1155 | } | |||
1156 | ||||
1157 | // FIXME: Reuse the LivePhysRegs utility for this. | |||
1158 | void ScheduleDAGInstrs::fixupKills(MachineBasicBlock *MBB) { | |||
1159 | DEBUG(dbgs() << "Fixup kills for BB#" << MBB->getNumber() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { dbgs() << "Fixup kills for BB#" << MBB->getNumber() << '\n'; } } while (0); | |||
1160 | ||||
1161 | LiveRegs.resize(TRI->getNumRegs()); | |||
1162 | BitVector killedRegs(TRI->getNumRegs()); | |||
1163 | ||||
1164 | startBlockForKills(MBB); | |||
1165 | ||||
1166 | // Examine block from end to start... | |||
1167 | unsigned Count = MBB->size(); | |||
1168 | for (MachineBasicBlock::iterator I = MBB->end(), E = MBB->begin(); | |||
1169 | I != E; --Count) { | |||
1170 | MachineInstr *MI = --I; | |||
1171 | if (MI->isDebugValue()) | |||
1172 | continue; | |||
1173 | ||||
1174 | // Update liveness. Registers that are defed but not used in this | |||
1175 | // instruction are now dead. Mark register and all subregs as they | |||
1176 | // are completely defined. | |||
1177 | for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { | |||
1178 | MachineOperand &MO = MI->getOperand(i); | |||
1179 | if (MO.isRegMask()) | |||
1180 | LiveRegs.clearBitsNotInMask(MO.getRegMask()); | |||
1181 | if (!MO.isReg()) continue; | |||
1182 | unsigned Reg = MO.getReg(); | |||
1183 | if (Reg == 0) continue; | |||
1184 | if (!MO.isDef()) continue; | |||
1185 | // Ignore two-addr defs. | |||
1186 | if (MI->isRegTiedToUseOperand(i)) continue; | |||
1187 | ||||
1188 | // Repeat for reg and all subregs. | |||
1189 | for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true); | |||
1190 | SubRegs.isValid(); ++SubRegs) | |||
1191 | LiveRegs.reset(*SubRegs); | |||
1192 | } | |||
1193 | ||||
1194 | // Examine all used registers and set/clear kill flag. When a | |||
1195 | // register is used multiple times we only set the kill flag on | |||
1196 | // the first use. Don't set kill flags on undef operands. | |||
1197 | killedRegs.reset(); | |||
1198 | for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { | |||
1199 | MachineOperand &MO = MI->getOperand(i); | |||
1200 | if (!MO.isReg() || !MO.isUse() || MO.isUndef()) continue; | |||
1201 | unsigned Reg = MO.getReg(); | |||
1202 | if ((Reg == 0) || MRI.isReserved(Reg)) continue; | |||
1203 | ||||
1204 | bool kill = false; | |||
1205 | if (!killedRegs.test(Reg)) { | |||
1206 | kill = true; | |||
1207 | // A register is not killed if any subregs are live... | |||
1208 | for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs) { | |||
1209 | if (LiveRegs.test(*SubRegs)) { | |||
1210 | kill = false; | |||
1211 | break; | |||
1212 | } | |||
1213 | } | |||
1214 | ||||
1215 | // If subreg is not live, then register is killed if it became | |||
1216 | // live in this instruction | |||
1217 | if (kill) | |||
1218 | kill = !LiveRegs.test(Reg); | |||
1219 | } | |||
1220 | ||||
1221 | if (MO.isKill() != kill) { | |||
1222 | DEBUG(dbgs() << "Fixing " << MO << " in ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { dbgs() << "Fixing " << MO << " in "; } } while (0); | |||
1223 | // Warning: toggleKillFlag may invalidate MO. | |||
1224 | toggleKillFlag(MI, MO); | |||
1225 | DEBUG(MI->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { MI->dump(); } } while (0); | |||
1226 | DEBUG(if (MI->getOpcode() == TargetOpcode::BUNDLE) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { if (MI->getOpcode() == TargetOpcode::BUNDLE ) { MachineBasicBlock::instr_iterator Begin = MI->getIterator (); MachineBasicBlock::instr_iterator End = getBundleEnd(MI); while (++Begin != End) do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType("misched")) { Begin->dump(); } } while (0); }; } } while (0) | |||
1227 | MachineBasicBlock::instr_iterator Begin = MI->getIterator();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { if (MI->getOpcode() == TargetOpcode::BUNDLE ) { MachineBasicBlock::instr_iterator Begin = MI->getIterator (); MachineBasicBlock::instr_iterator End = getBundleEnd(MI); while (++Begin != End) do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType("misched")) { Begin->dump(); } } while (0); }; } } while (0) | |||
1228 | MachineBasicBlock::instr_iterator End = getBundleEnd(MI);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { if (MI->getOpcode() == TargetOpcode::BUNDLE ) { MachineBasicBlock::instr_iterator Begin = MI->getIterator (); MachineBasicBlock::instr_iterator End = getBundleEnd(MI); while (++Begin != End) do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType("misched")) { Begin->dump(); } } while (0); }; } } while (0) | |||
1229 | while (++Begin != End)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { if (MI->getOpcode() == TargetOpcode::BUNDLE ) { MachineBasicBlock::instr_iterator Begin = MI->getIterator (); MachineBasicBlock::instr_iterator End = getBundleEnd(MI); while (++Begin != End) do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType("misched")) { Begin->dump(); } } while (0); }; } } while (0) | |||
1230 | DEBUG(Begin->dump());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { if (MI->getOpcode() == TargetOpcode::BUNDLE ) { MachineBasicBlock::instr_iterator Begin = MI->getIterator (); MachineBasicBlock::instr_iterator End = getBundleEnd(MI); while (++Begin != End) do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType("misched")) { Begin->dump(); } } while (0); }; } } while (0) | |||
1231 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { if (MI->getOpcode() == TargetOpcode::BUNDLE ) { MachineBasicBlock::instr_iterator Begin = MI->getIterator (); MachineBasicBlock::instr_iterator End = getBundleEnd(MI); while (++Begin != End) do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType("misched")) { Begin->dump(); } } while (0); }; } } while (0); | |||
1232 | } | |||
1233 | ||||
1234 | killedRegs.set(Reg); | |||
1235 | } | |||
1236 | ||||
1237 | // Mark any used register (that is not using undef) and subregs as | |||
1238 | // now live... | |||
1239 | for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { | |||
1240 | MachineOperand &MO = MI->getOperand(i); | |||
1241 | if (!MO.isReg() || !MO.isUse() || MO.isUndef()) continue; | |||
1242 | unsigned Reg = MO.getReg(); | |||
1243 | if ((Reg == 0) || MRI.isReserved(Reg)) continue; | |||
1244 | ||||
1245 | for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true); | |||
1246 | SubRegs.isValid(); ++SubRegs) | |||
1247 | LiveRegs.set(*SubRegs); | |||
1248 | } | |||
1249 | } | |||
1250 | } | |||
1251 | ||||
1252 | void ScheduleDAGInstrs::dumpNode(const SUnit *SU) const { | |||
1253 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
1254 | SU->getInstr()->dump(); | |||
1255 | #endif | |||
1256 | } | |||
1257 | ||||
1258 | std::string ScheduleDAGInstrs::getGraphNodeLabel(const SUnit *SU) const { | |||
1259 | std::string s; | |||
1260 | raw_string_ostream oss(s); | |||
1261 | if (SU == &EntrySU) | |||
1262 | oss << "<entry>"; | |||
1263 | else if (SU == &ExitSU) | |||
1264 | oss << "<exit>"; | |||
1265 | else | |||
1266 | SU->getInstr()->print(oss, /*SkipOpers=*/true); | |||
1267 | return oss.str(); | |||
1268 | } | |||
1269 | ||||
1270 | /// Return the basic block label. It is not necessarilly unique because a block | |||
1271 | /// contains multiple scheduling regions. But it is fine for visualization. | |||
1272 | std::string ScheduleDAGInstrs::getDAGName() const { | |||
1273 | return "dag." + BB->getFullName(); | |||
1274 | } | |||
1275 | ||||
1276 | //===----------------------------------------------------------------------===// | |||
1277 | // SchedDFSResult Implementation | |||
1278 | //===----------------------------------------------------------------------===// | |||
1279 | ||||
1280 | namespace llvm { | |||
1281 | /// \brief Internal state used to compute SchedDFSResult. | |||
1282 | class SchedDFSImpl { | |||
1283 | SchedDFSResult &R; | |||
1284 | ||||
1285 | /// Join DAG nodes into equivalence classes by their subtree. | |||
1286 | IntEqClasses SubtreeClasses; | |||
1287 | /// List PredSU, SuccSU pairs that represent data edges between subtrees. | |||
1288 | std::vector<std::pair<const SUnit*, const SUnit*> > ConnectionPairs; | |||
1289 | ||||
1290 | struct RootData { | |||
1291 | unsigned NodeID; | |||
1292 | unsigned ParentNodeID; // Parent node (member of the parent subtree). | |||
1293 | unsigned SubInstrCount; // Instr count in this tree only, not children. | |||
1294 | ||||
1295 | RootData(unsigned id): NodeID(id), | |||
1296 | ParentNodeID(SchedDFSResult::InvalidSubtreeID), | |||
1297 | SubInstrCount(0) {} | |||
1298 | ||||
1299 | unsigned getSparseSetIndex() const { return NodeID; } | |||
1300 | }; | |||
1301 | ||||
1302 | SparseSet<RootData> RootSet; | |||
1303 | ||||
1304 | public: | |||
1305 | SchedDFSImpl(SchedDFSResult &r): R(r), SubtreeClasses(R.DFSNodeData.size()) { | |||
1306 | RootSet.setUniverse(R.DFSNodeData.size()); | |||
1307 | } | |||
1308 | ||||
1309 | /// Return true if this node been visited by the DFS traversal. | |||
1310 | /// | |||
1311 | /// During visitPostorderNode the Node's SubtreeID is assigned to the Node | |||
1312 | /// ID. Later, SubtreeID is updated but remains valid. | |||
1313 | bool isVisited(const SUnit *SU) const { | |||
1314 | return R.DFSNodeData[SU->NodeNum].SubtreeID | |||
1315 | != SchedDFSResult::InvalidSubtreeID; | |||
1316 | } | |||
1317 | ||||
1318 | /// Initialize this node's instruction count. We don't need to flag the node | |||
1319 | /// visited until visitPostorder because the DAG cannot have cycles. | |||
1320 | void visitPreorder(const SUnit *SU) { | |||
1321 | R.DFSNodeData[SU->NodeNum].InstrCount = | |||
1322 | SU->getInstr()->isTransient() ? 0 : 1; | |||
1323 | } | |||
1324 | ||||
1325 | /// Called once for each node after all predecessors are visited. Revisit this | |||
1326 | /// node's predecessors and potentially join them now that we know the ILP of | |||
1327 | /// the other predecessors. | |||
1328 | void visitPostorderNode(const SUnit *SU) { | |||
1329 | // Mark this node as the root of a subtree. It may be joined with its | |||
1330 | // successors later. | |||
1331 | R.DFSNodeData[SU->NodeNum].SubtreeID = SU->NodeNum; | |||
1332 | RootData RData(SU->NodeNum); | |||
1333 | RData.SubInstrCount = SU->getInstr()->isTransient() ? 0 : 1; | |||
1334 | ||||
1335 | // If any predecessors are still in their own subtree, they either cannot be | |||
1336 | // joined or are large enough to remain separate. If this parent node's | |||
1337 | // total instruction count is not greater than a child subtree by at least | |||
1338 | // the subtree limit, then try to join it now since splitting subtrees is | |||
1339 | // only useful if multiple high-pressure paths are possible. | |||
1340 | unsigned InstrCount = R.DFSNodeData[SU->NodeNum].InstrCount; | |||
1341 | for (SUnit::const_pred_iterator | |||
1342 | PI = SU->Preds.begin(), PE = SU->Preds.end(); PI != PE; ++PI) { | |||
1343 | if (PI->getKind() != SDep::Data) | |||
1344 | continue; | |||
1345 | unsigned PredNum = PI->getSUnit()->NodeNum; | |||
1346 | if ((InstrCount - R.DFSNodeData[PredNum].InstrCount) < R.SubtreeLimit) | |||
1347 | joinPredSubtree(*PI, SU, /*CheckLimit=*/false); | |||
1348 | ||||
1349 | // Either link or merge the TreeData entry from the child to the parent. | |||
1350 | if (R.DFSNodeData[PredNum].SubtreeID == PredNum) { | |||
1351 | // If the predecessor's parent is invalid, this is a tree edge and the | |||
1352 | // current node is the parent. | |||
1353 | if (RootSet[PredNum].ParentNodeID == SchedDFSResult::InvalidSubtreeID) | |||
1354 | RootSet[PredNum].ParentNodeID = SU->NodeNum; | |||
1355 | } | |||
1356 | else if (RootSet.count(PredNum)) { | |||
1357 | // The predecessor is not a root, but is still in the root set. This | |||
1358 | // must be the new parent that it was just joined to. Note that | |||
1359 | // RootSet[PredNum].ParentNodeID may either be invalid or may still be | |||
1360 | // set to the original parent. | |||
1361 | RData.SubInstrCount += RootSet[PredNum].SubInstrCount; | |||
1362 | RootSet.erase(PredNum); | |||
1363 | } | |||
1364 | } | |||
1365 | RootSet[SU->NodeNum] = RData; | |||
1366 | } | |||
1367 | ||||
1368 | /// Called once for each tree edge after calling visitPostOrderNode on the | |||
1369 | /// predecessor. Increment the parent node's instruction count and | |||
1370 | /// preemptively join this subtree to its parent's if it is small enough. | |||
1371 | void visitPostorderEdge(const SDep &PredDep, const SUnit *Succ) { | |||
1372 | R.DFSNodeData[Succ->NodeNum].InstrCount | |||
1373 | += R.DFSNodeData[PredDep.getSUnit()->NodeNum].InstrCount; | |||
1374 | joinPredSubtree(PredDep, Succ); | |||
1375 | } | |||
1376 | ||||
1377 | /// Add a connection for cross edges. | |||
1378 | void visitCrossEdge(const SDep &PredDep, const SUnit *Succ) { | |||
1379 | ConnectionPairs.push_back(std::make_pair(PredDep.getSUnit(), Succ)); | |||
1380 | } | |||
1381 | ||||
1382 | /// Set each node's subtree ID to the representative ID and record connections | |||
1383 | /// between trees. | |||
1384 | void finalize() { | |||
1385 | SubtreeClasses.compress(); | |||
1386 | R.DFSTreeData.resize(SubtreeClasses.getNumClasses()); | |||
1387 | assert(SubtreeClasses.getNumClasses() == RootSet.size()((SubtreeClasses.getNumClasses() == RootSet.size() && "number of roots should match trees") ? static_cast<void> (0) : __assert_fail ("SubtreeClasses.getNumClasses() == RootSet.size() && \"number of roots should match trees\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 1388, __PRETTY_FUNCTION__)) | |||
1388 | && "number of roots should match trees")((SubtreeClasses.getNumClasses() == RootSet.size() && "number of roots should match trees") ? static_cast<void> (0) : __assert_fail ("SubtreeClasses.getNumClasses() == RootSet.size() && \"number of roots should match trees\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 1388, __PRETTY_FUNCTION__)); | |||
1389 | for (SparseSet<RootData>::const_iterator | |||
1390 | RI = RootSet.begin(), RE = RootSet.end(); RI != RE; ++RI) { | |||
1391 | unsigned TreeID = SubtreeClasses[RI->NodeID]; | |||
1392 | if (RI->ParentNodeID != SchedDFSResult::InvalidSubtreeID) | |||
1393 | R.DFSTreeData[TreeID].ParentTreeID = SubtreeClasses[RI->ParentNodeID]; | |||
1394 | R.DFSTreeData[TreeID].SubInstrCount = RI->SubInstrCount; | |||
1395 | // Note that SubInstrCount may be greater than InstrCount if we joined | |||
1396 | // subtrees across a cross edge. InstrCount will be attributed to the | |||
1397 | // original parent, while SubInstrCount will be attributed to the joined | |||
1398 | // parent. | |||
1399 | } | |||
1400 | R.SubtreeConnections.resize(SubtreeClasses.getNumClasses()); | |||
1401 | R.SubtreeConnectLevels.resize(SubtreeClasses.getNumClasses()); | |||
1402 | DEBUG(dbgs() << R.getNumSubtrees() << " subtrees:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { dbgs() << R.getNumSubtrees() << " subtrees:\n" ; } } while (0); | |||
1403 | for (unsigned Idx = 0, End = R.DFSNodeData.size(); Idx != End; ++Idx) { | |||
1404 | R.DFSNodeData[Idx].SubtreeID = SubtreeClasses[Idx]; | |||
1405 | DEBUG(dbgs() << " SU(" << Idx << ") in tree "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { dbgs() << " SU(" << Idx << ") in tree " << R.DFSNodeData[Idx].SubtreeID << '\n'; } } while (0) | |||
1406 | << R.DFSNodeData[Idx].SubtreeID << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { dbgs() << " SU(" << Idx << ") in tree " << R.DFSNodeData[Idx].SubtreeID << '\n'; } } while (0); | |||
1407 | } | |||
1408 | for (std::vector<std::pair<const SUnit*, const SUnit*> >::const_iterator | |||
1409 | I = ConnectionPairs.begin(), E = ConnectionPairs.end(); | |||
1410 | I != E; ++I) { | |||
1411 | unsigned PredTree = SubtreeClasses[I->first->NodeNum]; | |||
1412 | unsigned SuccTree = SubtreeClasses[I->second->NodeNum]; | |||
1413 | if (PredTree == SuccTree) | |||
1414 | continue; | |||
1415 | unsigned Depth = I->first->getDepth(); | |||
1416 | addConnection(PredTree, SuccTree, Depth); | |||
1417 | addConnection(SuccTree, PredTree, Depth); | |||
1418 | } | |||
1419 | } | |||
1420 | ||||
1421 | protected: | |||
1422 | /// Join the predecessor subtree with the successor that is its DFS | |||
1423 | /// parent. Apply some heuristics before joining. | |||
1424 | bool joinPredSubtree(const SDep &PredDep, const SUnit *Succ, | |||
1425 | bool CheckLimit = true) { | |||
1426 | assert(PredDep.getKind() == SDep::Data && "Subtrees are for data edges")((PredDep.getKind() == SDep::Data && "Subtrees are for data edges" ) ? static_cast<void> (0) : __assert_fail ("PredDep.getKind() == SDep::Data && \"Subtrees are for data edges\"" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 1426, __PRETTY_FUNCTION__)); | |||
1427 | ||||
1428 | // Check if the predecessor is already joined. | |||
1429 | const SUnit *PredSU = PredDep.getSUnit(); | |||
1430 | unsigned PredNum = PredSU->NodeNum; | |||
1431 | if (R.DFSNodeData[PredNum].SubtreeID != PredNum) | |||
1432 | return false; | |||
1433 | ||||
1434 | // Four is the magic number of successors before a node is considered a | |||
1435 | // pinch point. | |||
1436 | unsigned NumDataSucs = 0; | |||
1437 | for (SUnit::const_succ_iterator SI = PredSU->Succs.begin(), | |||
1438 | SE = PredSU->Succs.end(); SI != SE; ++SI) { | |||
1439 | if (SI->getKind() == SDep::Data) { | |||
1440 | if (++NumDataSucs >= 4) | |||
1441 | return false; | |||
1442 | } | |||
1443 | } | |||
1444 | if (CheckLimit && R.DFSNodeData[PredNum].InstrCount > R.SubtreeLimit) | |||
1445 | return false; | |||
1446 | R.DFSNodeData[PredNum].SubtreeID = Succ->NodeNum; | |||
1447 | SubtreeClasses.join(Succ->NodeNum, PredNum); | |||
1448 | return true; | |||
1449 | } | |||
1450 | ||||
1451 | /// Called by finalize() to record a connection between trees. | |||
1452 | void addConnection(unsigned FromTree, unsigned ToTree, unsigned Depth) { | |||
1453 | if (!Depth) | |||
1454 | return; | |||
1455 | ||||
1456 | do { | |||
1457 | SmallVectorImpl<SchedDFSResult::Connection> &Connections = | |||
1458 | R.SubtreeConnections[FromTree]; | |||
1459 | for (SmallVectorImpl<SchedDFSResult::Connection>::iterator | |||
1460 | I = Connections.begin(), E = Connections.end(); I != E; ++I) { | |||
1461 | if (I->TreeID == ToTree) { | |||
1462 | I->Level = std::max(I->Level, Depth); | |||
1463 | return; | |||
1464 | } | |||
1465 | } | |||
1466 | Connections.push_back(SchedDFSResult::Connection(ToTree, Depth)); | |||
1467 | FromTree = R.DFSTreeData[FromTree].ParentTreeID; | |||
1468 | } while (FromTree != SchedDFSResult::InvalidSubtreeID); | |||
1469 | } | |||
1470 | }; | |||
1471 | } // namespace llvm | |||
1472 | ||||
1473 | namespace { | |||
1474 | /// \brief Manage the stack used by a reverse depth-first search over the DAG. | |||
1475 | class SchedDAGReverseDFS { | |||
1476 | std::vector<std::pair<const SUnit*, SUnit::const_pred_iterator> > DFSStack; | |||
1477 | public: | |||
1478 | bool isComplete() const { return DFSStack.empty(); } | |||
1479 | ||||
1480 | void follow(const SUnit *SU) { | |||
1481 | DFSStack.push_back(std::make_pair(SU, SU->Preds.begin())); | |||
1482 | } | |||
1483 | void advance() { ++DFSStack.back().second; } | |||
1484 | ||||
1485 | const SDep *backtrack() { | |||
1486 | DFSStack.pop_back(); | |||
1487 | return DFSStack.empty() ? nullptr : std::prev(DFSStack.back().second); | |||
1488 | } | |||
1489 | ||||
1490 | const SUnit *getCurr() const { return DFSStack.back().first; } | |||
1491 | ||||
1492 | SUnit::const_pred_iterator getPred() const { return DFSStack.back().second; } | |||
1493 | ||||
1494 | SUnit::const_pred_iterator getPredEnd() const { | |||
1495 | return getCurr()->Preds.end(); | |||
1496 | } | |||
1497 | }; | |||
1498 | } // anonymous | |||
1499 | ||||
1500 | static bool hasDataSucc(const SUnit *SU) { | |||
1501 | for (SUnit::const_succ_iterator | |||
1502 | SI = SU->Succs.begin(), SE = SU->Succs.end(); SI != SE; ++SI) { | |||
1503 | if (SI->getKind() == SDep::Data && !SI->getSUnit()->isBoundaryNode()) | |||
1504 | return true; | |||
1505 | } | |||
1506 | return false; | |||
1507 | } | |||
1508 | ||||
1509 | /// Compute an ILP metric for all nodes in the subDAG reachable via depth-first | |||
1510 | /// search from this root. | |||
1511 | void SchedDFSResult::compute(ArrayRef<SUnit> SUnits) { | |||
1512 | if (!IsBottomUp) | |||
1513 | llvm_unreachable("Top-down ILP metric is unimplemnted")::llvm::llvm_unreachable_internal("Top-down ILP metric is unimplemnted" , "/tmp/buildd/llvm-toolchain-snapshot-3.8~svn254649/lib/CodeGen/ScheduleDAGInstrs.cpp" , 1513); | |||
1514 | ||||
1515 | SchedDFSImpl Impl(*this); | |||
1516 | for (ArrayRef<SUnit>::const_iterator | |||
1517 | SI = SUnits.begin(), SE = SUnits.end(); SI != SE; ++SI) { | |||
1518 | const SUnit *SU = &*SI; | |||
1519 | if (Impl.isVisited(SU) || hasDataSucc(SU)) | |||
1520 | continue; | |||
1521 | ||||
1522 | SchedDAGReverseDFS DFS; | |||
1523 | Impl.visitPreorder(SU); | |||
1524 | DFS.follow(SU); | |||
1525 | for (;;) { | |||
1526 | // Traverse the leftmost path as far as possible. | |||
1527 | while (DFS.getPred() != DFS.getPredEnd()) { | |||
1528 | const SDep &PredDep = *DFS.getPred(); | |||
1529 | DFS.advance(); | |||
1530 | // Ignore non-data edges. | |||
1531 | if (PredDep.getKind() != SDep::Data | |||
1532 | || PredDep.getSUnit()->isBoundaryNode()) { | |||
1533 | continue; | |||
1534 | } | |||
1535 | // An already visited edge is a cross edge, assuming an acyclic DAG. | |||
1536 | if (Impl.isVisited(PredDep.getSUnit())) { | |||
1537 | Impl.visitCrossEdge(PredDep, DFS.getCurr()); | |||
1538 | continue; | |||
1539 | } | |||
1540 | Impl.visitPreorder(PredDep.getSUnit()); | |||
1541 | DFS.follow(PredDep.getSUnit()); | |||
1542 | } | |||
1543 | // Visit the top of the stack in postorder and backtrack. | |||
1544 | const SUnit *Child = DFS.getCurr(); | |||
1545 | const SDep *PredDep = DFS.backtrack(); | |||
1546 | Impl.visitPostorderNode(Child); | |||
1547 | if (PredDep) | |||
1548 | Impl.visitPostorderEdge(*PredDep, DFS.getCurr()); | |||
1549 | if (DFS.isComplete()) | |||
1550 | break; | |||
1551 | } | |||
1552 | } | |||
1553 | Impl.finalize(); | |||
1554 | } | |||
1555 | ||||
1556 | /// The root of the given SubtreeID was just scheduled. For all subtrees | |||
1557 | /// connected to this tree, record the depth of the connection so that the | |||
1558 | /// nearest connected subtrees can be prioritized. | |||
1559 | void SchedDFSResult::scheduleTree(unsigned SubtreeID) { | |||
1560 | for (SmallVectorImpl<Connection>::const_iterator | |||
1561 | I = SubtreeConnections[SubtreeID].begin(), | |||
1562 | E = SubtreeConnections[SubtreeID].end(); I != E; ++I) { | |||
1563 | SubtreeConnectLevels[I->TreeID] = | |||
1564 | std::max(SubtreeConnectLevels[I->TreeID], I->Level); | |||
1565 | DEBUG(dbgs() << " Tree: " << I->TreeIDdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { dbgs() << " Tree: " << I->TreeID << " @" << SubtreeConnectLevels[I->TreeID] << '\n'; } } while (0) | |||
1566 | << " @" << SubtreeConnectLevels[I->TreeID] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("misched")) { dbgs() << " Tree: " << I->TreeID << " @" << SubtreeConnectLevels[I->TreeID] << '\n'; } } while (0); | |||
1567 | } | |||
1568 | } | |||
1569 | ||||
1570 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) | |||
1571 | void ILPValue::print(raw_ostream &OS) const { | |||
1572 | OS << InstrCount << " / " << Length << " = "; | |||
1573 | if (!Length) | |||
1574 | OS << "BADILP"; | |||
1575 | else | |||
1576 | OS << format("%g", ((double)InstrCount / Length)); | |||
1577 | } | |||
1578 | ||||
1579 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) | |||
1580 | void ILPValue::dump() const { | |||
1581 | dbgs() << *this << '\n'; | |||
1582 | } | |||
1583 | ||||
1584 | namespace llvm { | |||
1585 | ||||
1586 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) | |||
1587 | raw_ostream &operator<<(raw_ostream &OS, const ILPValue &Val) { | |||
1588 | Val.print(OS); | |||
1589 | return OS; | |||
1590 | } | |||
1591 | ||||
1592 | } // namespace llvm |