File: | build/source/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp |
Warning: | line 5675, column 15 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This transformation analyzes and transforms the induction variables (and | |||
10 | // computations derived from them) into forms suitable for efficient execution | |||
11 | // on the target. | |||
12 | // | |||
13 | // This pass performs a strength reduction on array references inside loops that | |||
14 | // have as one or more of their components the loop induction variable, it | |||
15 | // rewrites expressions to take advantage of scaled-index addressing modes | |||
16 | // available on the target, and it performs a variety of other optimizations | |||
17 | // related to loop induction variables. | |||
18 | // | |||
19 | // Terminology note: this code has a lot of handling for "post-increment" or | |||
20 | // "post-inc" users. This is not talking about post-increment addressing modes; | |||
21 | // it is instead talking about code like this: | |||
22 | // | |||
23 | // %i = phi [ 0, %entry ], [ %i.next, %latch ] | |||
24 | // ... | |||
25 | // %i.next = add %i, 1 | |||
26 | // %c = icmp eq %i.next, %n | |||
27 | // | |||
28 | // The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however | |||
29 | // it's useful to think about these as the same register, with some uses using | |||
30 | // the value of the register before the add and some using it after. In this | |||
31 | // example, the icmp is a post-increment user, since it uses %i.next, which is | |||
32 | // the value of the induction variable after the increment. The other common | |||
33 | // case of post-increment users is users outside the loop. | |||
34 | // | |||
35 | // TODO: More sophistication in the way Formulae are generated and filtered. | |||
36 | // | |||
37 | // TODO: Handle multiple loops at a time. | |||
38 | // | |||
39 | // TODO: Should the addressing mode BaseGV be changed to a ConstantExpr instead | |||
40 | // of a GlobalValue? | |||
41 | // | |||
42 | // TODO: When truncation is free, truncate ICmp users' operands to make it a | |||
43 | // smaller encoding (on x86 at least). | |||
44 | // | |||
45 | // TODO: When a negated register is used by an add (such as in a list of | |||
46 | // multiple base registers, or as the increment expression in an addrec), | |||
47 | // we may not actually need both reg and (-1 * reg) in registers; the | |||
48 | // negation can be implemented by using a sub instead of an add. The | |||
49 | // lack of support for taking this into consideration when making | |||
50 | // register pressure decisions is partly worked around by the "Special" | |||
51 | // use kind. | |||
52 | // | |||
53 | //===----------------------------------------------------------------------===// | |||
54 | ||||
55 | #include "llvm/Transforms/Scalar/LoopStrengthReduce.h" | |||
56 | #include "llvm/ADT/APInt.h" | |||
57 | #include "llvm/ADT/DenseMap.h" | |||
58 | #include "llvm/ADT/DenseSet.h" | |||
59 | #include "llvm/ADT/Hashing.h" | |||
60 | #include "llvm/ADT/PointerIntPair.h" | |||
61 | #include "llvm/ADT/STLExtras.h" | |||
62 | #include "llvm/ADT/SetVector.h" | |||
63 | #include "llvm/ADT/SmallBitVector.h" | |||
64 | #include "llvm/ADT/SmallPtrSet.h" | |||
65 | #include "llvm/ADT/SmallSet.h" | |||
66 | #include "llvm/ADT/SmallVector.h" | |||
67 | #include "llvm/ADT/Statistic.h" | |||
68 | #include "llvm/ADT/iterator_range.h" | |||
69 | #include "llvm/Analysis/AssumptionCache.h" | |||
70 | #include "llvm/Analysis/IVUsers.h" | |||
71 | #include "llvm/Analysis/LoopAnalysisManager.h" | |||
72 | #include "llvm/Analysis/LoopInfo.h" | |||
73 | #include "llvm/Analysis/LoopPass.h" | |||
74 | #include "llvm/Analysis/MemorySSA.h" | |||
75 | #include "llvm/Analysis/MemorySSAUpdater.h" | |||
76 | #include "llvm/Analysis/ScalarEvolution.h" | |||
77 | #include "llvm/Analysis/ScalarEvolutionExpressions.h" | |||
78 | #include "llvm/Analysis/ScalarEvolutionNormalization.h" | |||
79 | #include "llvm/Analysis/TargetLibraryInfo.h" | |||
80 | #include "llvm/Analysis/TargetTransformInfo.h" | |||
81 | #include "llvm/Analysis/ValueTracking.h" | |||
82 | #include "llvm/BinaryFormat/Dwarf.h" | |||
83 | #include "llvm/Config/llvm-config.h" | |||
84 | #include "llvm/IR/BasicBlock.h" | |||
85 | #include "llvm/IR/Constant.h" | |||
86 | #include "llvm/IR/Constants.h" | |||
87 | #include "llvm/IR/DebugInfoMetadata.h" | |||
88 | #include "llvm/IR/DerivedTypes.h" | |||
89 | #include "llvm/IR/Dominators.h" | |||
90 | #include "llvm/IR/GlobalValue.h" | |||
91 | #include "llvm/IR/IRBuilder.h" | |||
92 | #include "llvm/IR/InstrTypes.h" | |||
93 | #include "llvm/IR/Instruction.h" | |||
94 | #include "llvm/IR/Instructions.h" | |||
95 | #include "llvm/IR/IntrinsicInst.h" | |||
96 | #include "llvm/IR/Module.h" | |||
97 | #include "llvm/IR/Operator.h" | |||
98 | #include "llvm/IR/PassManager.h" | |||
99 | #include "llvm/IR/Type.h" | |||
100 | #include "llvm/IR/Use.h" | |||
101 | #include "llvm/IR/User.h" | |||
102 | #include "llvm/IR/Value.h" | |||
103 | #include "llvm/IR/ValueHandle.h" | |||
104 | #include "llvm/InitializePasses.h" | |||
105 | #include "llvm/Pass.h" | |||
106 | #include "llvm/Support/Casting.h" | |||
107 | #include "llvm/Support/CommandLine.h" | |||
108 | #include "llvm/Support/Compiler.h" | |||
109 | #include "llvm/Support/Debug.h" | |||
110 | #include "llvm/Support/ErrorHandling.h" | |||
111 | #include "llvm/Support/MathExtras.h" | |||
112 | #include "llvm/Support/raw_ostream.h" | |||
113 | #include "llvm/Transforms/Scalar.h" | |||
114 | #include "llvm/Transforms/Utils.h" | |||
115 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" | |||
116 | #include "llvm/Transforms/Utils/Local.h" | |||
117 | #include "llvm/Transforms/Utils/LoopUtils.h" | |||
118 | #include "llvm/Transforms/Utils/ScalarEvolutionExpander.h" | |||
119 | #include <algorithm> | |||
120 | #include <cassert> | |||
121 | #include <cstddef> | |||
122 | #include <cstdint> | |||
123 | #include <iterator> | |||
124 | #include <limits> | |||
125 | #include <map> | |||
126 | #include <numeric> | |||
127 | #include <optional> | |||
128 | #include <utility> | |||
129 | ||||
130 | using namespace llvm; | |||
131 | ||||
132 | #define DEBUG_TYPE"loop-reduce" "loop-reduce" | |||
133 | ||||
134 | /// MaxIVUsers is an arbitrary threshold that provides an early opportunity for | |||
135 | /// bail out. This threshold is far beyond the number of users that LSR can | |||
136 | /// conceivably solve, so it should not affect generated code, but catches the | |||
137 | /// worst cases before LSR burns too much compile time and stack space. | |||
138 | static const unsigned MaxIVUsers = 200; | |||
139 | ||||
140 | /// Limit the size of expression that SCEV-based salvaging will attempt to | |||
141 | /// translate into a DIExpression. | |||
142 | /// Choose a maximum size such that debuginfo is not excessively increased and | |||
143 | /// the salvaging is not too expensive for the compiler. | |||
144 | static const unsigned MaxSCEVSalvageExpressionSize = 64; | |||
145 | ||||
146 | // Cleanup congruent phis after LSR phi expansion. | |||
147 | static cl::opt<bool> EnablePhiElim( | |||
148 | "enable-lsr-phielim", cl::Hidden, cl::init(true), | |||
149 | cl::desc("Enable LSR phi elimination")); | |||
150 | ||||
151 | // The flag adds instruction count to solutions cost comparison. | |||
152 | static cl::opt<bool> InsnsCost( | |||
153 | "lsr-insns-cost", cl::Hidden, cl::init(true), | |||
154 | cl::desc("Add instruction count to a LSR cost model")); | |||
155 | ||||
156 | // Flag to choose how to narrow complex lsr solution | |||
157 | static cl::opt<bool> LSRExpNarrow( | |||
158 | "lsr-exp-narrow", cl::Hidden, cl::init(false), | |||
159 | cl::desc("Narrow LSR complex solution using" | |||
160 | " expectation of registers number")); | |||
161 | ||||
162 | // Flag to narrow search space by filtering non-optimal formulae with | |||
163 | // the same ScaledReg and Scale. | |||
164 | static cl::opt<bool> FilterSameScaledReg( | |||
165 | "lsr-filter-same-scaled-reg", cl::Hidden, cl::init(true), | |||
166 | cl::desc("Narrow LSR search space by filtering non-optimal formulae" | |||
167 | " with the same ScaledReg and Scale")); | |||
168 | ||||
169 | static cl::opt<TTI::AddressingModeKind> PreferredAddresingMode( | |||
170 | "lsr-preferred-addressing-mode", cl::Hidden, cl::init(TTI::AMK_None), | |||
171 | cl::desc("A flag that overrides the target's preferred addressing mode."), | |||
172 | cl::values(clEnumValN(TTI::AMK_None,llvm::cl::OptionEnumValue { "none", int(TTI::AMK_None), "Don't prefer any addressing mode" } | |||
173 | "none",llvm::cl::OptionEnumValue { "none", int(TTI::AMK_None), "Don't prefer any addressing mode" } | |||
174 | "Don't prefer any addressing mode")llvm::cl::OptionEnumValue { "none", int(TTI::AMK_None), "Don't prefer any addressing mode" }, | |||
175 | clEnumValN(TTI::AMK_PreIndexed,llvm::cl::OptionEnumValue { "preindexed", int(TTI::AMK_PreIndexed ), "Prefer pre-indexed addressing mode" } | |||
176 | "preindexed",llvm::cl::OptionEnumValue { "preindexed", int(TTI::AMK_PreIndexed ), "Prefer pre-indexed addressing mode" } | |||
177 | "Prefer pre-indexed addressing mode")llvm::cl::OptionEnumValue { "preindexed", int(TTI::AMK_PreIndexed ), "Prefer pre-indexed addressing mode" }, | |||
178 | clEnumValN(TTI::AMK_PostIndexed,llvm::cl::OptionEnumValue { "postindexed", int(TTI::AMK_PostIndexed ), "Prefer post-indexed addressing mode" } | |||
179 | "postindexed",llvm::cl::OptionEnumValue { "postindexed", int(TTI::AMK_PostIndexed ), "Prefer post-indexed addressing mode" } | |||
180 | "Prefer post-indexed addressing mode")llvm::cl::OptionEnumValue { "postindexed", int(TTI::AMK_PostIndexed ), "Prefer post-indexed addressing mode" })); | |||
181 | ||||
182 | static cl::opt<unsigned> ComplexityLimit( | |||
183 | "lsr-complexity-limit", cl::Hidden, | |||
184 | cl::init(std::numeric_limits<uint16_t>::max()), | |||
185 | cl::desc("LSR search space complexity limit")); | |||
186 | ||||
187 | static cl::opt<unsigned> SetupCostDepthLimit( | |||
188 | "lsr-setupcost-depth-limit", cl::Hidden, cl::init(7), | |||
189 | cl::desc("The limit on recursion depth for LSRs setup cost")); | |||
190 | ||||
191 | static cl::opt<bool> AllowTerminatingConditionFoldingAfterLSR( | |||
192 | "lsr-term-fold", cl::Hidden, cl::init(false), | |||
193 | cl::desc("Attempt to replace primary IV with other IV.")); | |||
194 | ||||
195 | static cl::opt<bool> AllowDropSolutionIfLessProfitable( | |||
196 | "lsr-drop-solution", cl::Hidden, cl::init(false), | |||
197 | cl::desc("Attempt to drop solution if it is less profitable")); | |||
198 | ||||
199 | STATISTIC(NumTermFold,static llvm::Statistic NumTermFold = {"loop-reduce", "NumTermFold" , "Number of terminating condition fold recognized and performed" } | |||
200 | "Number of terminating condition fold recognized and performed")static llvm::Statistic NumTermFold = {"loop-reduce", "NumTermFold" , "Number of terminating condition fold recognized and performed" }; | |||
201 | ||||
202 | #ifndef NDEBUG | |||
203 | // Stress test IV chain generation. | |||
204 | static cl::opt<bool> StressIVChain( | |||
205 | "stress-ivchain", cl::Hidden, cl::init(false), | |||
206 | cl::desc("Stress test LSR IV chains")); | |||
207 | #else | |||
208 | static bool StressIVChain = false; | |||
209 | #endif | |||
210 | ||||
211 | namespace { | |||
212 | ||||
213 | struct MemAccessTy { | |||
214 | /// Used in situations where the accessed memory type is unknown. | |||
215 | static const unsigned UnknownAddressSpace = | |||
216 | std::numeric_limits<unsigned>::max(); | |||
217 | ||||
218 | Type *MemTy = nullptr; | |||
219 | unsigned AddrSpace = UnknownAddressSpace; | |||
220 | ||||
221 | MemAccessTy() = default; | |||
222 | MemAccessTy(Type *Ty, unsigned AS) : MemTy(Ty), AddrSpace(AS) {} | |||
223 | ||||
224 | bool operator==(MemAccessTy Other) const { | |||
225 | return MemTy == Other.MemTy && AddrSpace == Other.AddrSpace; | |||
226 | } | |||
227 | ||||
228 | bool operator!=(MemAccessTy Other) const { return !(*this == Other); } | |||
229 | ||||
230 | static MemAccessTy getUnknown(LLVMContext &Ctx, | |||
231 | unsigned AS = UnknownAddressSpace) { | |||
232 | return MemAccessTy(Type::getVoidTy(Ctx), AS); | |||
233 | } | |||
234 | ||||
235 | Type *getType() { return MemTy; } | |||
236 | }; | |||
237 | ||||
238 | /// This class holds data which is used to order reuse candidates. | |||
239 | class RegSortData { | |||
240 | public: | |||
241 | /// This represents the set of LSRUse indices which reference | |||
242 | /// a particular register. | |||
243 | SmallBitVector UsedByIndices; | |||
244 | ||||
245 | void print(raw_ostream &OS) const; | |||
246 | void dump() const; | |||
247 | }; | |||
248 | ||||
249 | } // end anonymous namespace | |||
250 | ||||
251 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
252 | void RegSortData::print(raw_ostream &OS) const { | |||
253 | OS << "[NumUses=" << UsedByIndices.count() << ']'; | |||
254 | } | |||
255 | ||||
256 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void RegSortData::dump() const { | |||
257 | print(errs()); errs() << '\n'; | |||
258 | } | |||
259 | #endif | |||
260 | ||||
261 | namespace { | |||
262 | ||||
263 | /// Map register candidates to information about how they are used. | |||
264 | class RegUseTracker { | |||
265 | using RegUsesTy = DenseMap<const SCEV *, RegSortData>; | |||
266 | ||||
267 | RegUsesTy RegUsesMap; | |||
268 | SmallVector<const SCEV *, 16> RegSequence; | |||
269 | ||||
270 | public: | |||
271 | void countRegister(const SCEV *Reg, size_t LUIdx); | |||
272 | void dropRegister(const SCEV *Reg, size_t LUIdx); | |||
273 | void swapAndDropUse(size_t LUIdx, size_t LastLUIdx); | |||
274 | ||||
275 | bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const; | |||
276 | ||||
277 | const SmallBitVector &getUsedByIndices(const SCEV *Reg) const; | |||
278 | ||||
279 | void clear(); | |||
280 | ||||
281 | using iterator = SmallVectorImpl<const SCEV *>::iterator; | |||
282 | using const_iterator = SmallVectorImpl<const SCEV *>::const_iterator; | |||
283 | ||||
284 | iterator begin() { return RegSequence.begin(); } | |||
285 | iterator end() { return RegSequence.end(); } | |||
286 | const_iterator begin() const { return RegSequence.begin(); } | |||
287 | const_iterator end() const { return RegSequence.end(); } | |||
288 | }; | |||
289 | ||||
290 | } // end anonymous namespace | |||
291 | ||||
292 | void | |||
293 | RegUseTracker::countRegister(const SCEV *Reg, size_t LUIdx) { | |||
294 | std::pair<RegUsesTy::iterator, bool> Pair = | |||
295 | RegUsesMap.insert(std::make_pair(Reg, RegSortData())); | |||
296 | RegSortData &RSD = Pair.first->second; | |||
297 | if (Pair.second) | |||
298 | RegSequence.push_back(Reg); | |||
299 | RSD.UsedByIndices.resize(std::max(RSD.UsedByIndices.size(), LUIdx + 1)); | |||
300 | RSD.UsedByIndices.set(LUIdx); | |||
301 | } | |||
302 | ||||
303 | void | |||
304 | RegUseTracker::dropRegister(const SCEV *Reg, size_t LUIdx) { | |||
305 | RegUsesTy::iterator It = RegUsesMap.find(Reg); | |||
306 | assert(It != RegUsesMap.end())(static_cast <bool> (It != RegUsesMap.end()) ? void (0) : __assert_fail ("It != RegUsesMap.end()", "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp" , 306, __extension__ __PRETTY_FUNCTION__)); | |||
307 | RegSortData &RSD = It->second; | |||
308 | assert(RSD.UsedByIndices.size() > LUIdx)(static_cast <bool> (RSD.UsedByIndices.size() > LUIdx ) ? void (0) : __assert_fail ("RSD.UsedByIndices.size() > LUIdx" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 308, __extension__ __PRETTY_FUNCTION__)); | |||
309 | RSD.UsedByIndices.reset(LUIdx); | |||
310 | } | |||
311 | ||||
312 | void | |||
313 | RegUseTracker::swapAndDropUse(size_t LUIdx, size_t LastLUIdx) { | |||
314 | assert(LUIdx <= LastLUIdx)(static_cast <bool> (LUIdx <= LastLUIdx) ? void (0) : __assert_fail ("LUIdx <= LastLUIdx", "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp" , 314, __extension__ __PRETTY_FUNCTION__)); | |||
315 | ||||
316 | // Update RegUses. The data structure is not optimized for this purpose; | |||
317 | // we must iterate through it and update each of the bit vectors. | |||
318 | for (auto &Pair : RegUsesMap) { | |||
319 | SmallBitVector &UsedByIndices = Pair.second.UsedByIndices; | |||
320 | if (LUIdx < UsedByIndices.size()) | |||
321 | UsedByIndices[LUIdx] = | |||
322 | LastLUIdx < UsedByIndices.size() ? UsedByIndices[LastLUIdx] : false; | |||
323 | UsedByIndices.resize(std::min(UsedByIndices.size(), LastLUIdx)); | |||
324 | } | |||
325 | } | |||
326 | ||||
327 | bool | |||
328 | RegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const { | |||
329 | RegUsesTy::const_iterator I = RegUsesMap.find(Reg); | |||
330 | if (I == RegUsesMap.end()) | |||
331 | return false; | |||
332 | const SmallBitVector &UsedByIndices = I->second.UsedByIndices; | |||
333 | int i = UsedByIndices.find_first(); | |||
334 | if (i == -1) return false; | |||
335 | if ((size_t)i != LUIdx) return true; | |||
336 | return UsedByIndices.find_next(i) != -1; | |||
337 | } | |||
338 | ||||
339 | const SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const { | |||
340 | RegUsesTy::const_iterator I = RegUsesMap.find(Reg); | |||
341 | assert(I != RegUsesMap.end() && "Unknown register!")(static_cast <bool> (I != RegUsesMap.end() && "Unknown register!" ) ? void (0) : __assert_fail ("I != RegUsesMap.end() && \"Unknown register!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 341, __extension__ __PRETTY_FUNCTION__)); | |||
342 | return I->second.UsedByIndices; | |||
343 | } | |||
344 | ||||
345 | void RegUseTracker::clear() { | |||
346 | RegUsesMap.clear(); | |||
347 | RegSequence.clear(); | |||
348 | } | |||
349 | ||||
350 | namespace { | |||
351 | ||||
352 | /// This class holds information that describes a formula for computing | |||
353 | /// satisfying a use. It may include broken-out immediates and scaled registers. | |||
354 | struct Formula { | |||
355 | /// Global base address used for complex addressing. | |||
356 | GlobalValue *BaseGV = nullptr; | |||
357 | ||||
358 | /// Base offset for complex addressing. | |||
359 | int64_t BaseOffset = 0; | |||
360 | ||||
361 | /// Whether any complex addressing has a base register. | |||
362 | bool HasBaseReg = false; | |||
363 | ||||
364 | /// The scale of any complex addressing. | |||
365 | int64_t Scale = 0; | |||
366 | ||||
367 | /// The list of "base" registers for this use. When this is non-empty. The | |||
368 | /// canonical representation of a formula is | |||
369 | /// 1. BaseRegs.size > 1 implies ScaledReg != NULL and | |||
370 | /// 2. ScaledReg != NULL implies Scale != 1 || !BaseRegs.empty(). | |||
371 | /// 3. The reg containing recurrent expr related with currect loop in the | |||
372 | /// formula should be put in the ScaledReg. | |||
373 | /// #1 enforces that the scaled register is always used when at least two | |||
374 | /// registers are needed by the formula: e.g., reg1 + reg2 is reg1 + 1 * reg2. | |||
375 | /// #2 enforces that 1 * reg is reg. | |||
376 | /// #3 ensures invariant regs with respect to current loop can be combined | |||
377 | /// together in LSR codegen. | |||
378 | /// This invariant can be temporarily broken while building a formula. | |||
379 | /// However, every formula inserted into the LSRInstance must be in canonical | |||
380 | /// form. | |||
381 | SmallVector<const SCEV *, 4> BaseRegs; | |||
382 | ||||
383 | /// The 'scaled' register for this use. This should be non-null when Scale is | |||
384 | /// not zero. | |||
385 | const SCEV *ScaledReg = nullptr; | |||
386 | ||||
387 | /// An additional constant offset which added near the use. This requires a | |||
388 | /// temporary register, but the offset itself can live in an add immediate | |||
389 | /// field rather than a register. | |||
390 | int64_t UnfoldedOffset = 0; | |||
391 | ||||
392 | Formula() = default; | |||
393 | ||||
394 | void initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE); | |||
395 | ||||
396 | bool isCanonical(const Loop &L) const; | |||
397 | ||||
398 | void canonicalize(const Loop &L); | |||
399 | ||||
400 | bool unscale(); | |||
401 | ||||
402 | bool hasZeroEnd() const; | |||
403 | ||||
404 | size_t getNumRegs() const; | |||
405 | Type *getType() const; | |||
406 | ||||
407 | void deleteBaseReg(const SCEV *&S); | |||
408 | ||||
409 | bool referencesReg(const SCEV *S) const; | |||
410 | bool hasRegsUsedByUsesOtherThan(size_t LUIdx, | |||
411 | const RegUseTracker &RegUses) const; | |||
412 | ||||
413 | void print(raw_ostream &OS) const; | |||
414 | void dump() const; | |||
415 | }; | |||
416 | ||||
417 | } // end anonymous namespace | |||
418 | ||||
419 | /// Recursion helper for initialMatch. | |||
420 | static void DoInitialMatch(const SCEV *S, Loop *L, | |||
421 | SmallVectorImpl<const SCEV *> &Good, | |||
422 | SmallVectorImpl<const SCEV *> &Bad, | |||
423 | ScalarEvolution &SE) { | |||
424 | // Collect expressions which properly dominate the loop header. | |||
425 | if (SE.properlyDominates(S, L->getHeader())) { | |||
426 | Good.push_back(S); | |||
427 | return; | |||
428 | } | |||
429 | ||||
430 | // Look at add operands. | |||
431 | if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { | |||
432 | for (const SCEV *S : Add->operands()) | |||
433 | DoInitialMatch(S, L, Good, Bad, SE); | |||
434 | return; | |||
435 | } | |||
436 | ||||
437 | // Look at addrec operands. | |||
438 | if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) | |||
439 | if (!AR->getStart()->isZero() && AR->isAffine()) { | |||
440 | DoInitialMatch(AR->getStart(), L, Good, Bad, SE); | |||
441 | DoInitialMatch(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0), | |||
442 | AR->getStepRecurrence(SE), | |||
443 | // FIXME: AR->getNoWrapFlags() | |||
444 | AR->getLoop(), SCEV::FlagAnyWrap), | |||
445 | L, Good, Bad, SE); | |||
446 | return; | |||
447 | } | |||
448 | ||||
449 | // Handle a multiplication by -1 (negation) if it didn't fold. | |||
450 | if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) | |||
451 | if (Mul->getOperand(0)->isAllOnesValue()) { | |||
452 | SmallVector<const SCEV *, 4> Ops(drop_begin(Mul->operands())); | |||
453 | const SCEV *NewMul = SE.getMulExpr(Ops); | |||
454 | ||||
455 | SmallVector<const SCEV *, 4> MyGood; | |||
456 | SmallVector<const SCEV *, 4> MyBad; | |||
457 | DoInitialMatch(NewMul, L, MyGood, MyBad, SE); | |||
458 | const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue( | |||
459 | SE.getEffectiveSCEVType(NewMul->getType()))); | |||
460 | for (const SCEV *S : MyGood) | |||
461 | Good.push_back(SE.getMulExpr(NegOne, S)); | |||
462 | for (const SCEV *S : MyBad) | |||
463 | Bad.push_back(SE.getMulExpr(NegOne, S)); | |||
464 | return; | |||
465 | } | |||
466 | ||||
467 | // Ok, we can't do anything interesting. Just stuff the whole thing into a | |||
468 | // register and hope for the best. | |||
469 | Bad.push_back(S); | |||
470 | } | |||
471 | ||||
472 | /// Incorporate loop-variant parts of S into this Formula, attempting to keep | |||
473 | /// all loop-invariant and loop-computable values in a single base register. | |||
474 | void Formula::initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) { | |||
475 | SmallVector<const SCEV *, 4> Good; | |||
476 | SmallVector<const SCEV *, 4> Bad; | |||
477 | DoInitialMatch(S, L, Good, Bad, SE); | |||
478 | if (!Good.empty()) { | |||
479 | const SCEV *Sum = SE.getAddExpr(Good); | |||
480 | if (!Sum->isZero()) | |||
481 | BaseRegs.push_back(Sum); | |||
482 | HasBaseReg = true; | |||
483 | } | |||
484 | if (!Bad.empty()) { | |||
485 | const SCEV *Sum = SE.getAddExpr(Bad); | |||
486 | if (!Sum->isZero()) | |||
487 | BaseRegs.push_back(Sum); | |||
488 | HasBaseReg = true; | |||
489 | } | |||
490 | canonicalize(*L); | |||
491 | } | |||
492 | ||||
493 | static bool containsAddRecDependentOnLoop(const SCEV *S, const Loop &L) { | |||
494 | return SCEVExprContains(S, [&L](const SCEV *S) { | |||
495 | return isa<SCEVAddRecExpr>(S) && (cast<SCEVAddRecExpr>(S)->getLoop() == &L); | |||
496 | }); | |||
497 | } | |||
498 | ||||
499 | /// Check whether or not this formula satisfies the canonical | |||
500 | /// representation. | |||
501 | /// \see Formula::BaseRegs. | |||
502 | bool Formula::isCanonical(const Loop &L) const { | |||
503 | if (!ScaledReg) | |||
504 | return BaseRegs.size() <= 1; | |||
505 | ||||
506 | if (Scale != 1) | |||
507 | return true; | |||
508 | ||||
509 | if (Scale == 1 && BaseRegs.empty()) | |||
510 | return false; | |||
511 | ||||
512 | if (containsAddRecDependentOnLoop(ScaledReg, L)) | |||
513 | return true; | |||
514 | ||||
515 | // If ScaledReg is not a recurrent expr, or it is but its loop is not current | |||
516 | // loop, meanwhile BaseRegs contains a recurrent expr reg related with current | |||
517 | // loop, we want to swap the reg in BaseRegs with ScaledReg. | |||
518 | return none_of(BaseRegs, [&L](const SCEV *S) { | |||
519 | return containsAddRecDependentOnLoop(S, L); | |||
520 | }); | |||
521 | } | |||
522 | ||||
523 | /// Helper method to morph a formula into its canonical representation. | |||
524 | /// \see Formula::BaseRegs. | |||
525 | /// Every formula having more than one base register, must use the ScaledReg | |||
526 | /// field. Otherwise, we would have to do special cases everywhere in LSR | |||
527 | /// to treat reg1 + reg2 + ... the same way as reg1 + 1*reg2 + ... | |||
528 | /// On the other hand, 1*reg should be canonicalized into reg. | |||
529 | void Formula::canonicalize(const Loop &L) { | |||
530 | if (isCanonical(L)) | |||
531 | return; | |||
532 | ||||
533 | if (BaseRegs.empty()) { | |||
534 | // No base reg? Use scale reg with scale = 1 as such. | |||
535 | assert(ScaledReg && "Expected 1*reg => reg")(static_cast <bool> (ScaledReg && "Expected 1*reg => reg" ) ? void (0) : __assert_fail ("ScaledReg && \"Expected 1*reg => reg\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 535, __extension__ __PRETTY_FUNCTION__)); | |||
536 | assert(Scale == 1 && "Expected 1*reg => reg")(static_cast <bool> (Scale == 1 && "Expected 1*reg => reg" ) ? void (0) : __assert_fail ("Scale == 1 && \"Expected 1*reg => reg\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 536, __extension__ __PRETTY_FUNCTION__)); | |||
537 | BaseRegs.push_back(ScaledReg); | |||
538 | Scale = 0; | |||
539 | ScaledReg = nullptr; | |||
540 | return; | |||
541 | } | |||
542 | ||||
543 | // Keep the invariant sum in BaseRegs and one of the variant sum in ScaledReg. | |||
544 | if (!ScaledReg) { | |||
545 | ScaledReg = BaseRegs.pop_back_val(); | |||
546 | Scale = 1; | |||
547 | } | |||
548 | ||||
549 | // If ScaledReg is an invariant with respect to L, find the reg from | |||
550 | // BaseRegs containing the recurrent expr related with Loop L. Swap the | |||
551 | // reg with ScaledReg. | |||
552 | if (!containsAddRecDependentOnLoop(ScaledReg, L)) { | |||
553 | auto I = find_if(BaseRegs, [&L](const SCEV *S) { | |||
554 | return containsAddRecDependentOnLoop(S, L); | |||
555 | }); | |||
556 | if (I != BaseRegs.end()) | |||
557 | std::swap(ScaledReg, *I); | |||
558 | } | |||
559 | assert(isCanonical(L) && "Failed to canonicalize?")(static_cast <bool> (isCanonical(L) && "Failed to canonicalize?" ) ? void (0) : __assert_fail ("isCanonical(L) && \"Failed to canonicalize?\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 559, __extension__ __PRETTY_FUNCTION__)); | |||
560 | } | |||
561 | ||||
562 | /// Get rid of the scale in the formula. | |||
563 | /// In other words, this method morphes reg1 + 1*reg2 into reg1 + reg2. | |||
564 | /// \return true if it was possible to get rid of the scale, false otherwise. | |||
565 | /// \note After this operation the formula may not be in the canonical form. | |||
566 | bool Formula::unscale() { | |||
567 | if (Scale != 1) | |||
568 | return false; | |||
569 | Scale = 0; | |||
570 | BaseRegs.push_back(ScaledReg); | |||
571 | ScaledReg = nullptr; | |||
572 | return true; | |||
573 | } | |||
574 | ||||
575 | bool Formula::hasZeroEnd() const { | |||
576 | if (UnfoldedOffset || BaseOffset) | |||
577 | return false; | |||
578 | if (BaseRegs.size() != 1 || ScaledReg) | |||
579 | return false; | |||
580 | return true; | |||
581 | } | |||
582 | ||||
583 | /// Return the total number of register operands used by this formula. This does | |||
584 | /// not include register uses implied by non-constant addrec strides. | |||
585 | size_t Formula::getNumRegs() const { | |||
586 | return !!ScaledReg + BaseRegs.size(); | |||
587 | } | |||
588 | ||||
589 | /// Return the type of this formula, if it has one, or null otherwise. This type | |||
590 | /// is meaningless except for the bit size. | |||
591 | Type *Formula::getType() const { | |||
592 | return !BaseRegs.empty() ? BaseRegs.front()->getType() : | |||
593 | ScaledReg ? ScaledReg->getType() : | |||
594 | BaseGV ? BaseGV->getType() : | |||
595 | nullptr; | |||
596 | } | |||
597 | ||||
598 | /// Delete the given base reg from the BaseRegs list. | |||
599 | void Formula::deleteBaseReg(const SCEV *&S) { | |||
600 | if (&S != &BaseRegs.back()) | |||
601 | std::swap(S, BaseRegs.back()); | |||
602 | BaseRegs.pop_back(); | |||
603 | } | |||
604 | ||||
605 | /// Test if this formula references the given register. | |||
606 | bool Formula::referencesReg(const SCEV *S) const { | |||
607 | return S == ScaledReg || is_contained(BaseRegs, S); | |||
608 | } | |||
609 | ||||
610 | /// Test whether this formula uses registers which are used by uses other than | |||
611 | /// the use with the given index. | |||
612 | bool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx, | |||
613 | const RegUseTracker &RegUses) const { | |||
614 | if (ScaledReg) | |||
615 | if (RegUses.isRegUsedByUsesOtherThan(ScaledReg, LUIdx)) | |||
616 | return true; | |||
617 | for (const SCEV *BaseReg : BaseRegs) | |||
618 | if (RegUses.isRegUsedByUsesOtherThan(BaseReg, LUIdx)) | |||
619 | return true; | |||
620 | return false; | |||
621 | } | |||
622 | ||||
623 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
624 | void Formula::print(raw_ostream &OS) const { | |||
625 | bool First = true; | |||
626 | if (BaseGV) { | |||
627 | if (!First) OS << " + "; else First = false; | |||
628 | BaseGV->printAsOperand(OS, /*PrintType=*/false); | |||
629 | } | |||
630 | if (BaseOffset != 0) { | |||
631 | if (!First) OS << " + "; else First = false; | |||
632 | OS << BaseOffset; | |||
633 | } | |||
634 | for (const SCEV *BaseReg : BaseRegs) { | |||
635 | if (!First) OS << " + "; else First = false; | |||
636 | OS << "reg(" << *BaseReg << ')'; | |||
637 | } | |||
638 | if (HasBaseReg && BaseRegs.empty()) { | |||
639 | if (!First) OS << " + "; else First = false; | |||
640 | OS << "**error: HasBaseReg**"; | |||
641 | } else if (!HasBaseReg && !BaseRegs.empty()) { | |||
642 | if (!First) OS << " + "; else First = false; | |||
643 | OS << "**error: !HasBaseReg**"; | |||
644 | } | |||
645 | if (Scale != 0) { | |||
646 | if (!First) OS << " + "; else First = false; | |||
647 | OS << Scale << "*reg("; | |||
648 | if (ScaledReg) | |||
649 | OS << *ScaledReg; | |||
650 | else | |||
651 | OS << "<unknown>"; | |||
652 | OS << ')'; | |||
653 | } | |||
654 | if (UnfoldedOffset != 0) { | |||
655 | if (!First) OS << " + "; | |||
656 | OS << "imm(" << UnfoldedOffset << ')'; | |||
657 | } | |||
658 | } | |||
659 | ||||
660 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void Formula::dump() const { | |||
661 | print(errs()); errs() << '\n'; | |||
662 | } | |||
663 | #endif | |||
664 | ||||
665 | /// Return true if the given addrec can be sign-extended without changing its | |||
666 | /// value. | |||
667 | static bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) { | |||
668 | Type *WideTy = | |||
669 | IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(AR->getType()) + 1); | |||
670 | return isa<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy)); | |||
671 | } | |||
672 | ||||
673 | /// Return true if the given add can be sign-extended without changing its | |||
674 | /// value. | |||
675 | static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) { | |||
676 | Type *WideTy = | |||
677 | IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(A->getType()) + 1); | |||
678 | return isa<SCEVAddExpr>(SE.getSignExtendExpr(A, WideTy)); | |||
679 | } | |||
680 | ||||
681 | /// Return true if the given mul can be sign-extended without changing its | |||
682 | /// value. | |||
683 | static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) { | |||
684 | Type *WideTy = | |||
685 | IntegerType::get(SE.getContext(), | |||
686 | SE.getTypeSizeInBits(M->getType()) * M->getNumOperands()); | |||
687 | return isa<SCEVMulExpr>(SE.getSignExtendExpr(M, WideTy)); | |||
688 | } | |||
689 | ||||
690 | /// Return an expression for LHS /s RHS, if it can be determined and if the | |||
691 | /// remainder is known to be zero, or null otherwise. If IgnoreSignificantBits | |||
692 | /// is true, expressions like (X * Y) /s Y are simplified to X, ignoring that | |||
693 | /// the multiplication may overflow, which is useful when the result will be | |||
694 | /// used in a context where the most significant bits are ignored. | |||
695 | static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS, | |||
696 | ScalarEvolution &SE, | |||
697 | bool IgnoreSignificantBits = false) { | |||
698 | // Handle the trivial case, which works for any SCEV type. | |||
699 | if (LHS == RHS) | |||
700 | return SE.getConstant(LHS->getType(), 1); | |||
701 | ||||
702 | // Handle a few RHS special cases. | |||
703 | const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS); | |||
704 | if (RC) { | |||
705 | const APInt &RA = RC->getAPInt(); | |||
706 | // Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do | |||
707 | // some folding. | |||
708 | if (RA.isAllOnes()) { | |||
709 | if (LHS->getType()->isPointerTy()) | |||
710 | return nullptr; | |||
711 | return SE.getMulExpr(LHS, RC); | |||
712 | } | |||
713 | // Handle x /s 1 as x. | |||
714 | if (RA == 1) | |||
715 | return LHS; | |||
716 | } | |||
717 | ||||
718 | // Check for a division of a constant by a constant. | |||
719 | if (const SCEVConstant *C = dyn_cast<SCEVConstant>(LHS)) { | |||
720 | if (!RC) | |||
721 | return nullptr; | |||
722 | const APInt &LA = C->getAPInt(); | |||
723 | const APInt &RA = RC->getAPInt(); | |||
724 | if (LA.srem(RA) != 0) | |||
725 | return nullptr; | |||
726 | return SE.getConstant(LA.sdiv(RA)); | |||
727 | } | |||
728 | ||||
729 | // Distribute the sdiv over addrec operands, if the addrec doesn't overflow. | |||
730 | if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) { | |||
731 | if ((IgnoreSignificantBits || isAddRecSExtable(AR, SE)) && AR->isAffine()) { | |||
732 | const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE, | |||
733 | IgnoreSignificantBits); | |||
734 | if (!Step) return nullptr; | |||
735 | const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE, | |||
736 | IgnoreSignificantBits); | |||
737 | if (!Start) return nullptr; | |||
738 | // FlagNW is independent of the start value, step direction, and is | |||
739 | // preserved with smaller magnitude steps. | |||
740 | // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) | |||
741 | return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagAnyWrap); | |||
742 | } | |||
743 | return nullptr; | |||
744 | } | |||
745 | ||||
746 | // Distribute the sdiv over add operands, if the add doesn't overflow. | |||
747 | if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(LHS)) { | |||
748 | if (IgnoreSignificantBits || isAddSExtable(Add, SE)) { | |||
749 | SmallVector<const SCEV *, 8> Ops; | |||
750 | for (const SCEV *S : Add->operands()) { | |||
751 | const SCEV *Op = getExactSDiv(S, RHS, SE, IgnoreSignificantBits); | |||
752 | if (!Op) return nullptr; | |||
753 | Ops.push_back(Op); | |||
754 | } | |||
755 | return SE.getAddExpr(Ops); | |||
756 | } | |||
757 | return nullptr; | |||
758 | } | |||
759 | ||||
760 | // Check for a multiply operand that we can pull RHS out of. | |||
761 | if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS)) { | |||
762 | if (IgnoreSignificantBits || isMulSExtable(Mul, SE)) { | |||
763 | // Handle special case C1*X*Y /s C2*X*Y. | |||
764 | if (const SCEVMulExpr *MulRHS = dyn_cast<SCEVMulExpr>(RHS)) { | |||
765 | if (IgnoreSignificantBits || isMulSExtable(MulRHS, SE)) { | |||
766 | const SCEVConstant *LC = dyn_cast<SCEVConstant>(Mul->getOperand(0)); | |||
767 | const SCEVConstant *RC = | |||
768 | dyn_cast<SCEVConstant>(MulRHS->getOperand(0)); | |||
769 | if (LC && RC) { | |||
770 | SmallVector<const SCEV *, 4> LOps(drop_begin(Mul->operands())); | |||
771 | SmallVector<const SCEV *, 4> ROps(drop_begin(MulRHS->operands())); | |||
772 | if (LOps == ROps) | |||
773 | return getExactSDiv(LC, RC, SE, IgnoreSignificantBits); | |||
774 | } | |||
775 | } | |||
776 | } | |||
777 | ||||
778 | SmallVector<const SCEV *, 4> Ops; | |||
779 | bool Found = false; | |||
780 | for (const SCEV *S : Mul->operands()) { | |||
781 | if (!Found) | |||
782 | if (const SCEV *Q = getExactSDiv(S, RHS, SE, | |||
783 | IgnoreSignificantBits)) { | |||
784 | S = Q; | |||
785 | Found = true; | |||
786 | } | |||
787 | Ops.push_back(S); | |||
788 | } | |||
789 | return Found ? SE.getMulExpr(Ops) : nullptr; | |||
790 | } | |||
791 | return nullptr; | |||
792 | } | |||
793 | ||||
794 | // Otherwise we don't know. | |||
795 | return nullptr; | |||
796 | } | |||
797 | ||||
798 | /// If S involves the addition of a constant integer value, return that integer | |||
799 | /// value, and mutate S to point to a new SCEV with that value excluded. | |||
800 | static int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) { | |||
801 | if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) { | |||
802 | if (C->getAPInt().getMinSignedBits() <= 64) { | |||
803 | S = SE.getConstant(C->getType(), 0); | |||
804 | return C->getValue()->getSExtValue(); | |||
805 | } | |||
806 | } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { | |||
807 | SmallVector<const SCEV *, 8> NewOps(Add->operands()); | |||
808 | int64_t Result = ExtractImmediate(NewOps.front(), SE); | |||
809 | if (Result != 0) | |||
810 | S = SE.getAddExpr(NewOps); | |||
811 | return Result; | |||
812 | } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { | |||
813 | SmallVector<const SCEV *, 8> NewOps(AR->operands()); | |||
814 | int64_t Result = ExtractImmediate(NewOps.front(), SE); | |||
815 | if (Result != 0) | |||
816 | S = SE.getAddRecExpr(NewOps, AR->getLoop(), | |||
817 | // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) | |||
818 | SCEV::FlagAnyWrap); | |||
819 | return Result; | |||
820 | } | |||
821 | return 0; | |||
822 | } | |||
823 | ||||
824 | /// If S involves the addition of a GlobalValue address, return that symbol, and | |||
825 | /// mutate S to point to a new SCEV with that value excluded. | |||
826 | static GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) { | |||
827 | if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { | |||
828 | if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) { | |||
829 | S = SE.getConstant(GV->getType(), 0); | |||
830 | return GV; | |||
831 | } | |||
832 | } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { | |||
833 | SmallVector<const SCEV *, 8> NewOps(Add->operands()); | |||
834 | GlobalValue *Result = ExtractSymbol(NewOps.back(), SE); | |||
835 | if (Result) | |||
836 | S = SE.getAddExpr(NewOps); | |||
837 | return Result; | |||
838 | } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { | |||
839 | SmallVector<const SCEV *, 8> NewOps(AR->operands()); | |||
840 | GlobalValue *Result = ExtractSymbol(NewOps.front(), SE); | |||
841 | if (Result) | |||
842 | S = SE.getAddRecExpr(NewOps, AR->getLoop(), | |||
843 | // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) | |||
844 | SCEV::FlagAnyWrap); | |||
845 | return Result; | |||
846 | } | |||
847 | return nullptr; | |||
848 | } | |||
849 | ||||
850 | /// Returns true if the specified instruction is using the specified value as an | |||
851 | /// address. | |||
852 | static bool isAddressUse(const TargetTransformInfo &TTI, | |||
853 | Instruction *Inst, Value *OperandVal) { | |||
854 | bool isAddress = isa<LoadInst>(Inst); | |||
855 | if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) { | |||
856 | if (SI->getPointerOperand() == OperandVal) | |||
857 | isAddress = true; | |||
858 | } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { | |||
859 | // Addressing modes can also be folded into prefetches and a variety | |||
860 | // of intrinsics. | |||
861 | switch (II->getIntrinsicID()) { | |||
862 | case Intrinsic::memset: | |||
863 | case Intrinsic::prefetch: | |||
864 | case Intrinsic::masked_load: | |||
865 | if (II->getArgOperand(0) == OperandVal) | |||
866 | isAddress = true; | |||
867 | break; | |||
868 | case Intrinsic::masked_store: | |||
869 | if (II->getArgOperand(1) == OperandVal) | |||
870 | isAddress = true; | |||
871 | break; | |||
872 | case Intrinsic::memmove: | |||
873 | case Intrinsic::memcpy: | |||
874 | if (II->getArgOperand(0) == OperandVal || | |||
875 | II->getArgOperand(1) == OperandVal) | |||
876 | isAddress = true; | |||
877 | break; | |||
878 | default: { | |||
879 | MemIntrinsicInfo IntrInfo; | |||
880 | if (TTI.getTgtMemIntrinsic(II, IntrInfo)) { | |||
881 | if (IntrInfo.PtrVal == OperandVal) | |||
882 | isAddress = true; | |||
883 | } | |||
884 | } | |||
885 | } | |||
886 | } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Inst)) { | |||
887 | if (RMW->getPointerOperand() == OperandVal) | |||
888 | isAddress = true; | |||
889 | } else if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Inst)) { | |||
890 | if (CmpX->getPointerOperand() == OperandVal) | |||
891 | isAddress = true; | |||
892 | } | |||
893 | return isAddress; | |||
894 | } | |||
895 | ||||
896 | /// Return the type of the memory being accessed. | |||
897 | static MemAccessTy getAccessType(const TargetTransformInfo &TTI, | |||
898 | Instruction *Inst, Value *OperandVal) { | |||
899 | MemAccessTy AccessTy(Inst->getType(), MemAccessTy::UnknownAddressSpace); | |||
900 | if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) { | |||
901 | AccessTy.MemTy = SI->getOperand(0)->getType(); | |||
902 | AccessTy.AddrSpace = SI->getPointerAddressSpace(); | |||
903 | } else if (const LoadInst *LI = dyn_cast<LoadInst>(Inst)) { | |||
904 | AccessTy.AddrSpace = LI->getPointerAddressSpace(); | |||
905 | } else if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Inst)) { | |||
906 | AccessTy.AddrSpace = RMW->getPointerAddressSpace(); | |||
907 | } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Inst)) { | |||
908 | AccessTy.AddrSpace = CmpX->getPointerAddressSpace(); | |||
909 | } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) { | |||
910 | switch (II->getIntrinsicID()) { | |||
911 | case Intrinsic::prefetch: | |||
912 | case Intrinsic::memset: | |||
913 | AccessTy.AddrSpace = II->getArgOperand(0)->getType()->getPointerAddressSpace(); | |||
914 | AccessTy.MemTy = OperandVal->getType(); | |||
915 | break; | |||
916 | case Intrinsic::memmove: | |||
917 | case Intrinsic::memcpy: | |||
918 | AccessTy.AddrSpace = OperandVal->getType()->getPointerAddressSpace(); | |||
919 | AccessTy.MemTy = OperandVal->getType(); | |||
920 | break; | |||
921 | case Intrinsic::masked_load: | |||
922 | AccessTy.AddrSpace = | |||
923 | II->getArgOperand(0)->getType()->getPointerAddressSpace(); | |||
924 | break; | |||
925 | case Intrinsic::masked_store: | |||
926 | AccessTy.MemTy = II->getOperand(0)->getType(); | |||
927 | AccessTy.AddrSpace = | |||
928 | II->getArgOperand(1)->getType()->getPointerAddressSpace(); | |||
929 | break; | |||
930 | default: { | |||
931 | MemIntrinsicInfo IntrInfo; | |||
932 | if (TTI.getTgtMemIntrinsic(II, IntrInfo) && IntrInfo.PtrVal) { | |||
933 | AccessTy.AddrSpace | |||
934 | = IntrInfo.PtrVal->getType()->getPointerAddressSpace(); | |||
935 | } | |||
936 | ||||
937 | break; | |||
938 | } | |||
939 | } | |||
940 | } | |||
941 | ||||
942 | // All pointers have the same requirements, so canonicalize them to an | |||
943 | // arbitrary pointer type to minimize variation. | |||
944 | if (PointerType *PTy = dyn_cast<PointerType>(AccessTy.MemTy)) | |||
945 | AccessTy.MemTy = PointerType::get(IntegerType::get(PTy->getContext(), 1), | |||
946 | PTy->getAddressSpace()); | |||
947 | ||||
948 | return AccessTy; | |||
949 | } | |||
950 | ||||
951 | /// Return true if this AddRec is already a phi in its loop. | |||
952 | static bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE) { | |||
953 | for (PHINode &PN : AR->getLoop()->getHeader()->phis()) { | |||
954 | if (SE.isSCEVable(PN.getType()) && | |||
955 | (SE.getEffectiveSCEVType(PN.getType()) == | |||
956 | SE.getEffectiveSCEVType(AR->getType())) && | |||
957 | SE.getSCEV(&PN) == AR) | |||
958 | return true; | |||
959 | } | |||
960 | return false; | |||
961 | } | |||
962 | ||||
963 | /// Check if expanding this expression is likely to incur significant cost. This | |||
964 | /// is tricky because SCEV doesn't track which expressions are actually computed | |||
965 | /// by the current IR. | |||
966 | /// | |||
967 | /// We currently allow expansion of IV increments that involve adds, | |||
968 | /// multiplication by constants, and AddRecs from existing phis. | |||
969 | /// | |||
970 | /// TODO: Allow UDivExpr if we can find an existing IV increment that is an | |||
971 | /// obvious multiple of the UDivExpr. | |||
972 | static bool isHighCostExpansion(const SCEV *S, | |||
973 | SmallPtrSetImpl<const SCEV*> &Processed, | |||
974 | ScalarEvolution &SE) { | |||
975 | // Zero/One operand expressions | |||
976 | switch (S->getSCEVType()) { | |||
977 | case scUnknown: | |||
978 | case scConstant: | |||
979 | return false; | |||
980 | case scTruncate: | |||
981 | return isHighCostExpansion(cast<SCEVTruncateExpr>(S)->getOperand(), | |||
982 | Processed, SE); | |||
983 | case scZeroExtend: | |||
984 | return isHighCostExpansion(cast<SCEVZeroExtendExpr>(S)->getOperand(), | |||
985 | Processed, SE); | |||
986 | case scSignExtend: | |||
987 | return isHighCostExpansion(cast<SCEVSignExtendExpr>(S)->getOperand(), | |||
988 | Processed, SE); | |||
989 | default: | |||
990 | break; | |||
991 | } | |||
992 | ||||
993 | if (!Processed.insert(S).second) | |||
994 | return false; | |||
995 | ||||
996 | if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { | |||
997 | for (const SCEV *S : Add->operands()) { | |||
998 | if (isHighCostExpansion(S, Processed, SE)) | |||
999 | return true; | |||
1000 | } | |||
1001 | return false; | |||
1002 | } | |||
1003 | ||||
1004 | if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { | |||
1005 | if (Mul->getNumOperands() == 2) { | |||
1006 | // Multiplication by a constant is ok | |||
1007 | if (isa<SCEVConstant>(Mul->getOperand(0))) | |||
1008 | return isHighCostExpansion(Mul->getOperand(1), Processed, SE); | |||
1009 | ||||
1010 | // If we have the value of one operand, check if an existing | |||
1011 | // multiplication already generates this expression. | |||
1012 | if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Mul->getOperand(1))) { | |||
1013 | Value *UVal = U->getValue(); | |||
1014 | for (User *UR : UVal->users()) { | |||
1015 | // If U is a constant, it may be used by a ConstantExpr. | |||
1016 | Instruction *UI = dyn_cast<Instruction>(UR); | |||
1017 | if (UI && UI->getOpcode() == Instruction::Mul && | |||
1018 | SE.isSCEVable(UI->getType())) { | |||
1019 | return SE.getSCEV(UI) == Mul; | |||
1020 | } | |||
1021 | } | |||
1022 | } | |||
1023 | } | |||
1024 | } | |||
1025 | ||||
1026 | if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { | |||
1027 | if (isExistingPhi(AR, SE)) | |||
1028 | return false; | |||
1029 | } | |||
1030 | ||||
1031 | // Fow now, consider any other type of expression (div/mul/min/max) high cost. | |||
1032 | return true; | |||
1033 | } | |||
1034 | ||||
1035 | namespace { | |||
1036 | ||||
1037 | class LSRUse; | |||
1038 | ||||
1039 | } // end anonymous namespace | |||
1040 | ||||
1041 | /// Check if the addressing mode defined by \p F is completely | |||
1042 | /// folded in \p LU at isel time. | |||
1043 | /// This includes address-mode folding and special icmp tricks. | |||
1044 | /// This function returns true if \p LU can accommodate what \p F | |||
1045 | /// defines and up to 1 base + 1 scaled + offset. | |||
1046 | /// In other words, if \p F has several base registers, this function may | |||
1047 | /// still return true. Therefore, users still need to account for | |||
1048 | /// additional base registers and/or unfolded offsets to derive an | |||
1049 | /// accurate cost model. | |||
1050 | static bool isAMCompletelyFolded(const TargetTransformInfo &TTI, | |||
1051 | const LSRUse &LU, const Formula &F); | |||
1052 | ||||
1053 | // Get the cost of the scaling factor used in F for LU. | |||
1054 | static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI, | |||
1055 | const LSRUse &LU, const Formula &F, | |||
1056 | const Loop &L); | |||
1057 | ||||
1058 | namespace { | |||
1059 | ||||
1060 | /// This class is used to measure and compare candidate formulae. | |||
1061 | class Cost { | |||
1062 | const Loop *L = nullptr; | |||
1063 | ScalarEvolution *SE = nullptr; | |||
1064 | const TargetTransformInfo *TTI = nullptr; | |||
1065 | TargetTransformInfo::LSRCost C; | |||
1066 | TTI::AddressingModeKind AMK = TTI::AMK_None; | |||
1067 | ||||
1068 | public: | |||
1069 | Cost() = delete; | |||
1070 | Cost(const Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI, | |||
1071 | TTI::AddressingModeKind AMK) : | |||
1072 | L(L), SE(&SE), TTI(&TTI), AMK(AMK) { | |||
1073 | C.Insns = 0; | |||
1074 | C.NumRegs = 0; | |||
1075 | C.AddRecCost = 0; | |||
1076 | C.NumIVMuls = 0; | |||
1077 | C.NumBaseAdds = 0; | |||
1078 | C.ImmCost = 0; | |||
1079 | C.SetupCost = 0; | |||
1080 | C.ScaleCost = 0; | |||
1081 | } | |||
1082 | ||||
1083 | bool isLess(const Cost &Other) const; | |||
1084 | ||||
1085 | void Lose(); | |||
1086 | ||||
1087 | #ifndef NDEBUG | |||
1088 | // Once any of the metrics loses, they must all remain losers. | |||
1089 | bool isValid() { | |||
1090 | return ((C.Insns | C.NumRegs | C.AddRecCost | C.NumIVMuls | C.NumBaseAdds | |||
1091 | | C.ImmCost | C.SetupCost | C.ScaleCost) != ~0u) | |||
1092 | || ((C.Insns & C.NumRegs & C.AddRecCost & C.NumIVMuls & C.NumBaseAdds | |||
1093 | & C.ImmCost & C.SetupCost & C.ScaleCost) == ~0u); | |||
1094 | } | |||
1095 | #endif | |||
1096 | ||||
1097 | bool isLoser() { | |||
1098 | assert(isValid() && "invalid cost")(static_cast <bool> (isValid() && "invalid cost" ) ? void (0) : __assert_fail ("isValid() && \"invalid cost\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 1098, __extension__ __PRETTY_FUNCTION__)); | |||
1099 | return C.NumRegs == ~0u; | |||
1100 | } | |||
1101 | ||||
1102 | void RateFormula(const Formula &F, | |||
1103 | SmallPtrSetImpl<const SCEV *> &Regs, | |||
1104 | const DenseSet<const SCEV *> &VisitedRegs, | |||
1105 | const LSRUse &LU, | |||
1106 | SmallPtrSetImpl<const SCEV *> *LoserRegs = nullptr); | |||
1107 | ||||
1108 | void print(raw_ostream &OS) const; | |||
1109 | void dump() const; | |||
1110 | ||||
1111 | private: | |||
1112 | void RateRegister(const Formula &F, const SCEV *Reg, | |||
1113 | SmallPtrSetImpl<const SCEV *> &Regs); | |||
1114 | void RatePrimaryRegister(const Formula &F, const SCEV *Reg, | |||
1115 | SmallPtrSetImpl<const SCEV *> &Regs, | |||
1116 | SmallPtrSetImpl<const SCEV *> *LoserRegs); | |||
1117 | }; | |||
1118 | ||||
1119 | /// An operand value in an instruction which is to be replaced with some | |||
1120 | /// equivalent, possibly strength-reduced, replacement. | |||
1121 | struct LSRFixup { | |||
1122 | /// The instruction which will be updated. | |||
1123 | Instruction *UserInst = nullptr; | |||
1124 | ||||
1125 | /// The operand of the instruction which will be replaced. The operand may be | |||
1126 | /// used more than once; every instance will be replaced. | |||
1127 | Value *OperandValToReplace = nullptr; | |||
1128 | ||||
1129 | /// If this user is to use the post-incremented value of an induction | |||
1130 | /// variable, this set is non-empty and holds the loops associated with the | |||
1131 | /// induction variable. | |||
1132 | PostIncLoopSet PostIncLoops; | |||
1133 | ||||
1134 | /// A constant offset to be added to the LSRUse expression. This allows | |||
1135 | /// multiple fixups to share the same LSRUse with different offsets, for | |||
1136 | /// example in an unrolled loop. | |||
1137 | int64_t Offset = 0; | |||
1138 | ||||
1139 | LSRFixup() = default; | |||
1140 | ||||
1141 | bool isUseFullyOutsideLoop(const Loop *L) const; | |||
1142 | ||||
1143 | void print(raw_ostream &OS) const; | |||
1144 | void dump() const; | |||
1145 | }; | |||
1146 | ||||
1147 | /// A DenseMapInfo implementation for holding DenseMaps and DenseSets of sorted | |||
1148 | /// SmallVectors of const SCEV*. | |||
1149 | struct UniquifierDenseMapInfo { | |||
1150 | static SmallVector<const SCEV *, 4> getEmptyKey() { | |||
1151 | SmallVector<const SCEV *, 4> V; | |||
1152 | V.push_back(reinterpret_cast<const SCEV *>(-1)); | |||
1153 | return V; | |||
1154 | } | |||
1155 | ||||
1156 | static SmallVector<const SCEV *, 4> getTombstoneKey() { | |||
1157 | SmallVector<const SCEV *, 4> V; | |||
1158 | V.push_back(reinterpret_cast<const SCEV *>(-2)); | |||
1159 | return V; | |||
1160 | } | |||
1161 | ||||
1162 | static unsigned getHashValue(const SmallVector<const SCEV *, 4> &V) { | |||
1163 | return static_cast<unsigned>(hash_combine_range(V.begin(), V.end())); | |||
1164 | } | |||
1165 | ||||
1166 | static bool isEqual(const SmallVector<const SCEV *, 4> &LHS, | |||
1167 | const SmallVector<const SCEV *, 4> &RHS) { | |||
1168 | return LHS == RHS; | |||
1169 | } | |||
1170 | }; | |||
1171 | ||||
1172 | /// This class holds the state that LSR keeps for each use in IVUsers, as well | |||
1173 | /// as uses invented by LSR itself. It includes information about what kinds of | |||
1174 | /// things can be folded into the user, information about the user itself, and | |||
1175 | /// information about how the use may be satisfied. TODO: Represent multiple | |||
1176 | /// users of the same expression in common? | |||
1177 | class LSRUse { | |||
1178 | DenseSet<SmallVector<const SCEV *, 4>, UniquifierDenseMapInfo> Uniquifier; | |||
1179 | ||||
1180 | public: | |||
1181 | /// An enum for a kind of use, indicating what types of scaled and immediate | |||
1182 | /// operands it might support. | |||
1183 | enum KindType { | |||
1184 | Basic, ///< A normal use, with no folding. | |||
1185 | Special, ///< A special case of basic, allowing -1 scales. | |||
1186 | Address, ///< An address use; folding according to TargetLowering | |||
1187 | ICmpZero ///< An equality icmp with both operands folded into one. | |||
1188 | // TODO: Add a generic icmp too? | |||
1189 | }; | |||
1190 | ||||
1191 | using SCEVUseKindPair = PointerIntPair<const SCEV *, 2, KindType>; | |||
1192 | ||||
1193 | KindType Kind; | |||
1194 | MemAccessTy AccessTy; | |||
1195 | ||||
1196 | /// The list of operands which are to be replaced. | |||
1197 | SmallVector<LSRFixup, 8> Fixups; | |||
1198 | ||||
1199 | /// Keep track of the min and max offsets of the fixups. | |||
1200 | int64_t MinOffset = std::numeric_limits<int64_t>::max(); | |||
1201 | int64_t MaxOffset = std::numeric_limits<int64_t>::min(); | |||
1202 | ||||
1203 | /// This records whether all of the fixups using this LSRUse are outside of | |||
1204 | /// the loop, in which case some special-case heuristics may be used. | |||
1205 | bool AllFixupsOutsideLoop = true; | |||
1206 | ||||
1207 | /// RigidFormula is set to true to guarantee that this use will be associated | |||
1208 | /// with a single formula--the one that initially matched. Some SCEV | |||
1209 | /// expressions cannot be expanded. This allows LSR to consider the registers | |||
1210 | /// used by those expressions without the need to expand them later after | |||
1211 | /// changing the formula. | |||
1212 | bool RigidFormula = false; | |||
1213 | ||||
1214 | /// This records the widest use type for any fixup using this | |||
1215 | /// LSRUse. FindUseWithSimilarFormula can't consider uses with different max | |||
1216 | /// fixup widths to be equivalent, because the narrower one may be relying on | |||
1217 | /// the implicit truncation to truncate away bogus bits. | |||
1218 | Type *WidestFixupType = nullptr; | |||
1219 | ||||
1220 | /// A list of ways to build a value that can satisfy this user. After the | |||
1221 | /// list is populated, one of these is selected heuristically and used to | |||
1222 | /// formulate a replacement for OperandValToReplace in UserInst. | |||
1223 | SmallVector<Formula, 12> Formulae; | |||
1224 | ||||
1225 | /// The set of register candidates used by all formulae in this LSRUse. | |||
1226 | SmallPtrSet<const SCEV *, 4> Regs; | |||
1227 | ||||
1228 | LSRUse(KindType K, MemAccessTy AT) : Kind(K), AccessTy(AT) {} | |||
1229 | ||||
1230 | LSRFixup &getNewFixup() { | |||
1231 | Fixups.push_back(LSRFixup()); | |||
1232 | return Fixups.back(); | |||
1233 | } | |||
1234 | ||||
1235 | void pushFixup(LSRFixup &f) { | |||
1236 | Fixups.push_back(f); | |||
1237 | if (f.Offset > MaxOffset) | |||
1238 | MaxOffset = f.Offset; | |||
1239 | if (f.Offset < MinOffset) | |||
1240 | MinOffset = f.Offset; | |||
1241 | } | |||
1242 | ||||
1243 | bool HasFormulaWithSameRegs(const Formula &F) const; | |||
1244 | float getNotSelectedProbability(const SCEV *Reg) const; | |||
1245 | bool InsertFormula(const Formula &F, const Loop &L); | |||
1246 | void DeleteFormula(Formula &F); | |||
1247 | void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses); | |||
1248 | ||||
1249 | void print(raw_ostream &OS) const; | |||
1250 | void dump() const; | |||
1251 | }; | |||
1252 | ||||
1253 | } // end anonymous namespace | |||
1254 | ||||
1255 | static bool isAMCompletelyFolded(const TargetTransformInfo &TTI, | |||
1256 | LSRUse::KindType Kind, MemAccessTy AccessTy, | |||
1257 | GlobalValue *BaseGV, int64_t BaseOffset, | |||
1258 | bool HasBaseReg, int64_t Scale, | |||
1259 | Instruction *Fixup = nullptr); | |||
1260 | ||||
1261 | static unsigned getSetupCost(const SCEV *Reg, unsigned Depth) { | |||
1262 | if (isa<SCEVUnknown>(Reg) || isa<SCEVConstant>(Reg)) | |||
1263 | return 1; | |||
1264 | if (Depth == 0) | |||
1265 | return 0; | |||
1266 | if (const auto *S = dyn_cast<SCEVAddRecExpr>(Reg)) | |||
1267 | return getSetupCost(S->getStart(), Depth - 1); | |||
1268 | if (auto S = dyn_cast<SCEVIntegralCastExpr>(Reg)) | |||
1269 | return getSetupCost(S->getOperand(), Depth - 1); | |||
1270 | if (auto S = dyn_cast<SCEVNAryExpr>(Reg)) | |||
1271 | return std::accumulate(S->operands().begin(), S->operands().end(), 0, | |||
1272 | [&](unsigned i, const SCEV *Reg) { | |||
1273 | return i + getSetupCost(Reg, Depth - 1); | |||
1274 | }); | |||
1275 | if (auto S = dyn_cast<SCEVUDivExpr>(Reg)) | |||
1276 | return getSetupCost(S->getLHS(), Depth - 1) + | |||
1277 | getSetupCost(S->getRHS(), Depth - 1); | |||
1278 | return 0; | |||
1279 | } | |||
1280 | ||||
1281 | /// Tally up interesting quantities from the given register. | |||
1282 | void Cost::RateRegister(const Formula &F, const SCEV *Reg, | |||
1283 | SmallPtrSetImpl<const SCEV *> &Regs) { | |||
1284 | if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Reg)) { | |||
1285 | // If this is an addrec for another loop, it should be an invariant | |||
1286 | // with respect to L since L is the innermost loop (at least | |||
1287 | // for now LSR only handles innermost loops). | |||
1288 | if (AR->getLoop() != L) { | |||
1289 | // If the AddRec exists, consider it's register free and leave it alone. | |||
1290 | if (isExistingPhi(AR, *SE) && AMK != TTI::AMK_PostIndexed) | |||
1291 | return; | |||
1292 | ||||
1293 | // It is bad to allow LSR for current loop to add induction variables | |||
1294 | // for its sibling loops. | |||
1295 | if (!AR->getLoop()->contains(L)) { | |||
1296 | Lose(); | |||
1297 | return; | |||
1298 | } | |||
1299 | ||||
1300 | // Otherwise, it will be an invariant with respect to Loop L. | |||
1301 | ++C.NumRegs; | |||
1302 | return; | |||
1303 | } | |||
1304 | ||||
1305 | unsigned LoopCost = 1; | |||
1306 | if (TTI->isIndexedLoadLegal(TTI->MIM_PostInc, AR->getType()) || | |||
1307 | TTI->isIndexedStoreLegal(TTI->MIM_PostInc, AR->getType())) { | |||
1308 | ||||
1309 | // If the step size matches the base offset, we could use pre-indexed | |||
1310 | // addressing. | |||
1311 | if (AMK == TTI::AMK_PreIndexed) { | |||
1312 | if (auto *Step = dyn_cast<SCEVConstant>(AR->getStepRecurrence(*SE))) | |||
1313 | if (Step->getAPInt() == F.BaseOffset) | |||
1314 | LoopCost = 0; | |||
1315 | } else if (AMK == TTI::AMK_PostIndexed) { | |||
1316 | const SCEV *LoopStep = AR->getStepRecurrence(*SE); | |||
1317 | if (isa<SCEVConstant>(LoopStep)) { | |||
1318 | const SCEV *LoopStart = AR->getStart(); | |||
1319 | if (!isa<SCEVConstant>(LoopStart) && | |||
1320 | SE->isLoopInvariant(LoopStart, L)) | |||
1321 | LoopCost = 0; | |||
1322 | } | |||
1323 | } | |||
1324 | } | |||
1325 | C.AddRecCost += LoopCost; | |||
1326 | ||||
1327 | // Add the step value register, if it needs one. | |||
1328 | // TODO: The non-affine case isn't precisely modeled here. | |||
1329 | if (!AR->isAffine() || !isa<SCEVConstant>(AR->getOperand(1))) { | |||
1330 | if (!Regs.count(AR->getOperand(1))) { | |||
1331 | RateRegister(F, AR->getOperand(1), Regs); | |||
1332 | if (isLoser()) | |||
1333 | return; | |||
1334 | } | |||
1335 | } | |||
1336 | } | |||
1337 | ++C.NumRegs; | |||
1338 | ||||
1339 | // Rough heuristic; favor registers which don't require extra setup | |||
1340 | // instructions in the preheader. | |||
1341 | C.SetupCost += getSetupCost(Reg, SetupCostDepthLimit); | |||
1342 | // Ensure we don't, even with the recusion limit, produce invalid costs. | |||
1343 | C.SetupCost = std::min<unsigned>(C.SetupCost, 1 << 16); | |||
1344 | ||||
1345 | C.NumIVMuls += isa<SCEVMulExpr>(Reg) && | |||
1346 | SE->hasComputableLoopEvolution(Reg, L); | |||
1347 | } | |||
1348 | ||||
1349 | /// Record this register in the set. If we haven't seen it before, rate | |||
1350 | /// it. Optional LoserRegs provides a way to declare any formula that refers to | |||
1351 | /// one of those regs an instant loser. | |||
1352 | void Cost::RatePrimaryRegister(const Formula &F, const SCEV *Reg, | |||
1353 | SmallPtrSetImpl<const SCEV *> &Regs, | |||
1354 | SmallPtrSetImpl<const SCEV *> *LoserRegs) { | |||
1355 | if (LoserRegs && LoserRegs->count(Reg)) { | |||
1356 | Lose(); | |||
1357 | return; | |||
1358 | } | |||
1359 | if (Regs.insert(Reg).second) { | |||
1360 | RateRegister(F, Reg, Regs); | |||
1361 | if (LoserRegs && isLoser()) | |||
1362 | LoserRegs->insert(Reg); | |||
1363 | } | |||
1364 | } | |||
1365 | ||||
1366 | void Cost::RateFormula(const Formula &F, | |||
1367 | SmallPtrSetImpl<const SCEV *> &Regs, | |||
1368 | const DenseSet<const SCEV *> &VisitedRegs, | |||
1369 | const LSRUse &LU, | |||
1370 | SmallPtrSetImpl<const SCEV *> *LoserRegs) { | |||
1371 | if (isLoser()) | |||
1372 | return; | |||
1373 | assert(F.isCanonical(*L) && "Cost is accurate only for canonical formula")(static_cast <bool> (F.isCanonical(*L) && "Cost is accurate only for canonical formula" ) ? void (0) : __assert_fail ("F.isCanonical(*L) && \"Cost is accurate only for canonical formula\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 1373, __extension__ __PRETTY_FUNCTION__)); | |||
1374 | // Tally up the registers. | |||
1375 | unsigned PrevAddRecCost = C.AddRecCost; | |||
1376 | unsigned PrevNumRegs = C.NumRegs; | |||
1377 | unsigned PrevNumBaseAdds = C.NumBaseAdds; | |||
1378 | if (const SCEV *ScaledReg = F.ScaledReg) { | |||
1379 | if (VisitedRegs.count(ScaledReg)) { | |||
1380 | Lose(); | |||
1381 | return; | |||
1382 | } | |||
1383 | RatePrimaryRegister(F, ScaledReg, Regs, LoserRegs); | |||
1384 | if (isLoser()) | |||
1385 | return; | |||
1386 | } | |||
1387 | for (const SCEV *BaseReg : F.BaseRegs) { | |||
1388 | if (VisitedRegs.count(BaseReg)) { | |||
1389 | Lose(); | |||
1390 | return; | |||
1391 | } | |||
1392 | RatePrimaryRegister(F, BaseReg, Regs, LoserRegs); | |||
1393 | if (isLoser()) | |||
1394 | return; | |||
1395 | } | |||
1396 | ||||
1397 | // Determine how many (unfolded) adds we'll need inside the loop. | |||
1398 | size_t NumBaseParts = F.getNumRegs(); | |||
1399 | if (NumBaseParts > 1) | |||
1400 | // Do not count the base and a possible second register if the target | |||
1401 | // allows to fold 2 registers. | |||
1402 | C.NumBaseAdds += | |||
1403 | NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(*TTI, LU, F))); | |||
1404 | C.NumBaseAdds += (F.UnfoldedOffset != 0); | |||
1405 | ||||
1406 | // Accumulate non-free scaling amounts. | |||
1407 | C.ScaleCost += *getScalingFactorCost(*TTI, LU, F, *L).getValue(); | |||
1408 | ||||
1409 | // Tally up the non-zero immediates. | |||
1410 | for (const LSRFixup &Fixup : LU.Fixups) { | |||
1411 | int64_t O = Fixup.Offset; | |||
1412 | int64_t Offset = (uint64_t)O + F.BaseOffset; | |||
1413 | if (F.BaseGV) | |||
1414 | C.ImmCost += 64; // Handle symbolic values conservatively. | |||
1415 | // TODO: This should probably be the pointer size. | |||
1416 | else if (Offset != 0) | |||
1417 | C.ImmCost += APInt(64, Offset, true).getMinSignedBits(); | |||
1418 | ||||
1419 | // Check with target if this offset with this instruction is | |||
1420 | // specifically not supported. | |||
1421 | if (LU.Kind == LSRUse::Address && Offset != 0 && | |||
1422 | !isAMCompletelyFolded(*TTI, LSRUse::Address, LU.AccessTy, F.BaseGV, | |||
1423 | Offset, F.HasBaseReg, F.Scale, Fixup.UserInst)) | |||
1424 | C.NumBaseAdds++; | |||
1425 | } | |||
1426 | ||||
1427 | // If we don't count instruction cost exit here. | |||
1428 | if (!InsnsCost) { | |||
1429 | assert(isValid() && "invalid cost")(static_cast <bool> (isValid() && "invalid cost" ) ? void (0) : __assert_fail ("isValid() && \"invalid cost\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 1429, __extension__ __PRETTY_FUNCTION__)); | |||
1430 | return; | |||
1431 | } | |||
1432 | ||||
1433 | // Treat every new register that exceeds TTI.getNumberOfRegisters() - 1 as | |||
1434 | // additional instruction (at least fill). | |||
1435 | // TODO: Need distinguish register class? | |||
1436 | unsigned TTIRegNum = TTI->getNumberOfRegisters( | |||
1437 | TTI->getRegisterClassForType(false, F.getType())) - 1; | |||
1438 | if (C.NumRegs > TTIRegNum) { | |||
1439 | // Cost already exceeded TTIRegNum, then only newly added register can add | |||
1440 | // new instructions. | |||
1441 | if (PrevNumRegs > TTIRegNum) | |||
1442 | C.Insns += (C.NumRegs - PrevNumRegs); | |||
1443 | else | |||
1444 | C.Insns += (C.NumRegs - TTIRegNum); | |||
1445 | } | |||
1446 | ||||
1447 | // If ICmpZero formula ends with not 0, it could not be replaced by | |||
1448 | // just add or sub. We'll need to compare final result of AddRec. | |||
1449 | // That means we'll need an additional instruction. But if the target can | |||
1450 | // macro-fuse a compare with a branch, don't count this extra instruction. | |||
1451 | // For -10 + {0, +, 1}: | |||
1452 | // i = i + 1; | |||
1453 | // cmp i, 10 | |||
1454 | // | |||
1455 | // For {-10, +, 1}: | |||
1456 | // i = i + 1; | |||
1457 | if (LU.Kind == LSRUse::ICmpZero && !F.hasZeroEnd() && | |||
1458 | !TTI->canMacroFuseCmp()) | |||
1459 | C.Insns++; | |||
1460 | // Each new AddRec adds 1 instruction to calculation. | |||
1461 | C.Insns += (C.AddRecCost - PrevAddRecCost); | |||
1462 | ||||
1463 | // BaseAdds adds instructions for unfolded registers. | |||
1464 | if (LU.Kind != LSRUse::ICmpZero) | |||
1465 | C.Insns += C.NumBaseAdds - PrevNumBaseAdds; | |||
1466 | assert(isValid() && "invalid cost")(static_cast <bool> (isValid() && "invalid cost" ) ? void (0) : __assert_fail ("isValid() && \"invalid cost\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 1466, __extension__ __PRETTY_FUNCTION__)); | |||
1467 | } | |||
1468 | ||||
1469 | /// Set this cost to a losing value. | |||
1470 | void Cost::Lose() { | |||
1471 | C.Insns = std::numeric_limits<unsigned>::max(); | |||
1472 | C.NumRegs = std::numeric_limits<unsigned>::max(); | |||
1473 | C.AddRecCost = std::numeric_limits<unsigned>::max(); | |||
1474 | C.NumIVMuls = std::numeric_limits<unsigned>::max(); | |||
1475 | C.NumBaseAdds = std::numeric_limits<unsigned>::max(); | |||
1476 | C.ImmCost = std::numeric_limits<unsigned>::max(); | |||
1477 | C.SetupCost = std::numeric_limits<unsigned>::max(); | |||
1478 | C.ScaleCost = std::numeric_limits<unsigned>::max(); | |||
1479 | } | |||
1480 | ||||
1481 | /// Choose the lower cost. | |||
1482 | bool Cost::isLess(const Cost &Other) const { | |||
1483 | if (InsnsCost.getNumOccurrences() > 0 && InsnsCost && | |||
1484 | C.Insns != Other.C.Insns) | |||
1485 | return C.Insns < Other.C.Insns; | |||
1486 | return TTI->isLSRCostLess(C, Other.C); | |||
1487 | } | |||
1488 | ||||
1489 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
1490 | void Cost::print(raw_ostream &OS) const { | |||
1491 | if (InsnsCost) | |||
1492 | OS << C.Insns << " instruction" << (C.Insns == 1 ? " " : "s "); | |||
1493 | OS << C.NumRegs << " reg" << (C.NumRegs == 1 ? "" : "s"); | |||
1494 | if (C.AddRecCost != 0) | |||
1495 | OS << ", with addrec cost " << C.AddRecCost; | |||
1496 | if (C.NumIVMuls != 0) | |||
1497 | OS << ", plus " << C.NumIVMuls << " IV mul" | |||
1498 | << (C.NumIVMuls == 1 ? "" : "s"); | |||
1499 | if (C.NumBaseAdds != 0) | |||
1500 | OS << ", plus " << C.NumBaseAdds << " base add" | |||
1501 | << (C.NumBaseAdds == 1 ? "" : "s"); | |||
1502 | if (C.ScaleCost != 0) | |||
1503 | OS << ", plus " << C.ScaleCost << " scale cost"; | |||
1504 | if (C.ImmCost != 0) | |||
1505 | OS << ", plus " << C.ImmCost << " imm cost"; | |||
1506 | if (C.SetupCost != 0) | |||
1507 | OS << ", plus " << C.SetupCost << " setup cost"; | |||
1508 | } | |||
1509 | ||||
1510 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void Cost::dump() const { | |||
1511 | print(errs()); errs() << '\n'; | |||
1512 | } | |||
1513 | #endif | |||
1514 | ||||
1515 | /// Test whether this fixup always uses its value outside of the given loop. | |||
1516 | bool LSRFixup::isUseFullyOutsideLoop(const Loop *L) const { | |||
1517 | // PHI nodes use their value in their incoming blocks. | |||
1518 | if (const PHINode *PN = dyn_cast<PHINode>(UserInst)) { | |||
1519 | for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) | |||
1520 | if (PN->getIncomingValue(i) == OperandValToReplace && | |||
1521 | L->contains(PN->getIncomingBlock(i))) | |||
1522 | return false; | |||
1523 | return true; | |||
1524 | } | |||
1525 | ||||
1526 | return !L->contains(UserInst); | |||
1527 | } | |||
1528 | ||||
1529 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
1530 | void LSRFixup::print(raw_ostream &OS) const { | |||
1531 | OS << "UserInst="; | |||
1532 | // Store is common and interesting enough to be worth special-casing. | |||
1533 | if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) { | |||
1534 | OS << "store "; | |||
1535 | Store->getOperand(0)->printAsOperand(OS, /*PrintType=*/false); | |||
1536 | } else if (UserInst->getType()->isVoidTy()) | |||
1537 | OS << UserInst->getOpcodeName(); | |||
1538 | else | |||
1539 | UserInst->printAsOperand(OS, /*PrintType=*/false); | |||
1540 | ||||
1541 | OS << ", OperandValToReplace="; | |||
1542 | OperandValToReplace->printAsOperand(OS, /*PrintType=*/false); | |||
1543 | ||||
1544 | for (const Loop *PIL : PostIncLoops) { | |||
1545 | OS << ", PostIncLoop="; | |||
1546 | PIL->getHeader()->printAsOperand(OS, /*PrintType=*/false); | |||
1547 | } | |||
1548 | ||||
1549 | if (Offset != 0) | |||
1550 | OS << ", Offset=" << Offset; | |||
1551 | } | |||
1552 | ||||
1553 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void LSRFixup::dump() const { | |||
1554 | print(errs()); errs() << '\n'; | |||
1555 | } | |||
1556 | #endif | |||
1557 | ||||
1558 | /// Test whether this use as a formula which has the same registers as the given | |||
1559 | /// formula. | |||
1560 | bool LSRUse::HasFormulaWithSameRegs(const Formula &F) const { | |||
1561 | SmallVector<const SCEV *, 4> Key = F.BaseRegs; | |||
1562 | if (F.ScaledReg) Key.push_back(F.ScaledReg); | |||
1563 | // Unstable sort by host order ok, because this is only used for uniquifying. | |||
1564 | llvm::sort(Key); | |||
1565 | return Uniquifier.count(Key); | |||
1566 | } | |||
1567 | ||||
1568 | /// The function returns a probability of selecting formula without Reg. | |||
1569 | float LSRUse::getNotSelectedProbability(const SCEV *Reg) const { | |||
1570 | unsigned FNum = 0; | |||
1571 | for (const Formula &F : Formulae) | |||
1572 | if (F.referencesReg(Reg)) | |||
1573 | FNum++; | |||
1574 | return ((float)(Formulae.size() - FNum)) / Formulae.size(); | |||
1575 | } | |||
1576 | ||||
1577 | /// If the given formula has not yet been inserted, add it to the list, and | |||
1578 | /// return true. Return false otherwise. The formula must be in canonical form. | |||
1579 | bool LSRUse::InsertFormula(const Formula &F, const Loop &L) { | |||
1580 | assert(F.isCanonical(L) && "Invalid canonical representation")(static_cast <bool> (F.isCanonical(L) && "Invalid canonical representation" ) ? void (0) : __assert_fail ("F.isCanonical(L) && \"Invalid canonical representation\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 1580, __extension__ __PRETTY_FUNCTION__)); | |||
1581 | ||||
1582 | if (!Formulae.empty() && RigidFormula) | |||
1583 | return false; | |||
1584 | ||||
1585 | SmallVector<const SCEV *, 4> Key = F.BaseRegs; | |||
1586 | if (F.ScaledReg) Key.push_back(F.ScaledReg); | |||
1587 | // Unstable sort by host order ok, because this is only used for uniquifying. | |||
1588 | llvm::sort(Key); | |||
1589 | ||||
1590 | if (!Uniquifier.insert(Key).second) | |||
1591 | return false; | |||
1592 | ||||
1593 | // Using a register to hold the value of 0 is not profitable. | |||
1594 | assert((!F.ScaledReg || !F.ScaledReg->isZero()) &&(static_cast <bool> ((!F.ScaledReg || !F.ScaledReg-> isZero()) && "Zero allocated in a scaled register!") ? void (0) : __assert_fail ("(!F.ScaledReg || !F.ScaledReg->isZero()) && \"Zero allocated in a scaled register!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 1595, __extension__ __PRETTY_FUNCTION__)) | |||
1595 | "Zero allocated in a scaled register!")(static_cast <bool> ((!F.ScaledReg || !F.ScaledReg-> isZero()) && "Zero allocated in a scaled register!") ? void (0) : __assert_fail ("(!F.ScaledReg || !F.ScaledReg->isZero()) && \"Zero allocated in a scaled register!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 1595, __extension__ __PRETTY_FUNCTION__)); | |||
1596 | #ifndef NDEBUG | |||
1597 | for (const SCEV *BaseReg : F.BaseRegs) | |||
1598 | assert(!BaseReg->isZero() && "Zero allocated in a base register!")(static_cast <bool> (!BaseReg->isZero() && "Zero allocated in a base register!" ) ? void (0) : __assert_fail ("!BaseReg->isZero() && \"Zero allocated in a base register!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 1598, __extension__ __PRETTY_FUNCTION__)); | |||
1599 | #endif | |||
1600 | ||||
1601 | // Add the formula to the list. | |||
1602 | Formulae.push_back(F); | |||
1603 | ||||
1604 | // Record registers now being used by this use. | |||
1605 | Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end()); | |||
1606 | if (F.ScaledReg) | |||
1607 | Regs.insert(F.ScaledReg); | |||
1608 | ||||
1609 | return true; | |||
1610 | } | |||
1611 | ||||
1612 | /// Remove the given formula from this use's list. | |||
1613 | void LSRUse::DeleteFormula(Formula &F) { | |||
1614 | if (&F != &Formulae.back()) | |||
1615 | std::swap(F, Formulae.back()); | |||
1616 | Formulae.pop_back(); | |||
1617 | } | |||
1618 | ||||
1619 | /// Recompute the Regs field, and update RegUses. | |||
1620 | void LSRUse::RecomputeRegs(size_t LUIdx, RegUseTracker &RegUses) { | |||
1621 | // Now that we've filtered out some formulae, recompute the Regs set. | |||
1622 | SmallPtrSet<const SCEV *, 4> OldRegs = std::move(Regs); | |||
1623 | Regs.clear(); | |||
1624 | for (const Formula &F : Formulae) { | |||
1625 | if (F.ScaledReg) Regs.insert(F.ScaledReg); | |||
1626 | Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end()); | |||
1627 | } | |||
1628 | ||||
1629 | // Update the RegTracker. | |||
1630 | for (const SCEV *S : OldRegs) | |||
1631 | if (!Regs.count(S)) | |||
1632 | RegUses.dropRegister(S, LUIdx); | |||
1633 | } | |||
1634 | ||||
1635 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
1636 | void LSRUse::print(raw_ostream &OS) const { | |||
1637 | OS << "LSR Use: Kind="; | |||
1638 | switch (Kind) { | |||
1639 | case Basic: OS << "Basic"; break; | |||
1640 | case Special: OS << "Special"; break; | |||
1641 | case ICmpZero: OS << "ICmpZero"; break; | |||
1642 | case Address: | |||
1643 | OS << "Address of "; | |||
1644 | if (AccessTy.MemTy->isPointerTy()) | |||
1645 | OS << "pointer"; // the full pointer type could be really verbose | |||
1646 | else { | |||
1647 | OS << *AccessTy.MemTy; | |||
1648 | } | |||
1649 | ||||
1650 | OS << " in addrspace(" << AccessTy.AddrSpace << ')'; | |||
1651 | } | |||
1652 | ||||
1653 | OS << ", Offsets={"; | |||
1654 | bool NeedComma = false; | |||
1655 | for (const LSRFixup &Fixup : Fixups) { | |||
1656 | if (NeedComma) OS << ','; | |||
1657 | OS << Fixup.Offset; | |||
1658 | NeedComma = true; | |||
1659 | } | |||
1660 | OS << '}'; | |||
1661 | ||||
1662 | if (AllFixupsOutsideLoop) | |||
1663 | OS << ", all-fixups-outside-loop"; | |||
1664 | ||||
1665 | if (WidestFixupType) | |||
1666 | OS << ", widest fixup type: " << *WidestFixupType; | |||
1667 | } | |||
1668 | ||||
1669 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void LSRUse::dump() const { | |||
1670 | print(errs()); errs() << '\n'; | |||
1671 | } | |||
1672 | #endif | |||
1673 | ||||
1674 | static bool isAMCompletelyFolded(const TargetTransformInfo &TTI, | |||
1675 | LSRUse::KindType Kind, MemAccessTy AccessTy, | |||
1676 | GlobalValue *BaseGV, int64_t BaseOffset, | |||
1677 | bool HasBaseReg, int64_t Scale, | |||
1678 | Instruction *Fixup/*= nullptr*/) { | |||
1679 | switch (Kind) { | |||
1680 | case LSRUse::Address: | |||
1681 | return TTI.isLegalAddressingMode(AccessTy.MemTy, BaseGV, BaseOffset, | |||
1682 | HasBaseReg, Scale, AccessTy.AddrSpace, Fixup); | |||
1683 | ||||
1684 | case LSRUse::ICmpZero: | |||
1685 | // There's not even a target hook for querying whether it would be legal to | |||
1686 | // fold a GV into an ICmp. | |||
1687 | if (BaseGV) | |||
1688 | return false; | |||
1689 | ||||
1690 | // ICmp only has two operands; don't allow more than two non-trivial parts. | |||
1691 | if (Scale != 0 && HasBaseReg && BaseOffset != 0) | |||
1692 | return false; | |||
1693 | ||||
1694 | // ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale by | |||
1695 | // putting the scaled register in the other operand of the icmp. | |||
1696 | if (Scale != 0 && Scale != -1) | |||
1697 | return false; | |||
1698 | ||||
1699 | // If we have low-level target information, ask the target if it can fold an | |||
1700 | // integer immediate on an icmp. | |||
1701 | if (BaseOffset != 0) { | |||
1702 | // We have one of: | |||
1703 | // ICmpZero BaseReg + BaseOffset => ICmp BaseReg, -BaseOffset | |||
1704 | // ICmpZero -1*ScaleReg + BaseOffset => ICmp ScaleReg, BaseOffset | |||
1705 | // Offs is the ICmp immediate. | |||
1706 | if (Scale == 0) | |||
1707 | // The cast does the right thing with | |||
1708 | // std::numeric_limits<int64_t>::min(). | |||
1709 | BaseOffset = -(uint64_t)BaseOffset; | |||
1710 | return TTI.isLegalICmpImmediate(BaseOffset); | |||
1711 | } | |||
1712 | ||||
1713 | // ICmpZero BaseReg + -1*ScaleReg => ICmp BaseReg, ScaleReg | |||
1714 | return true; | |||
1715 | ||||
1716 | case LSRUse::Basic: | |||
1717 | // Only handle single-register values. | |||
1718 | return !BaseGV && Scale == 0 && BaseOffset == 0; | |||
1719 | ||||
1720 | case LSRUse::Special: | |||
1721 | // Special case Basic to handle -1 scales. | |||
1722 | return !BaseGV && (Scale == 0 || Scale == -1) && BaseOffset == 0; | |||
1723 | } | |||
1724 | ||||
1725 | llvm_unreachable("Invalid LSRUse Kind!")::llvm::llvm_unreachable_internal("Invalid LSRUse Kind!", "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp" , 1725); | |||
1726 | } | |||
1727 | ||||
1728 | static bool isAMCompletelyFolded(const TargetTransformInfo &TTI, | |||
1729 | int64_t MinOffset, int64_t MaxOffset, | |||
1730 | LSRUse::KindType Kind, MemAccessTy AccessTy, | |||
1731 | GlobalValue *BaseGV, int64_t BaseOffset, | |||
1732 | bool HasBaseReg, int64_t Scale) { | |||
1733 | // Check for overflow. | |||
1734 | if (((int64_t)((uint64_t)BaseOffset + MinOffset) > BaseOffset) != | |||
1735 | (MinOffset > 0)) | |||
1736 | return false; | |||
1737 | MinOffset = (uint64_t)BaseOffset + MinOffset; | |||
1738 | if (((int64_t)((uint64_t)BaseOffset + MaxOffset) > BaseOffset) != | |||
1739 | (MaxOffset > 0)) | |||
1740 | return false; | |||
1741 | MaxOffset = (uint64_t)BaseOffset + MaxOffset; | |||
1742 | ||||
1743 | return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, MinOffset, | |||
1744 | HasBaseReg, Scale) && | |||
1745 | isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, MaxOffset, | |||
1746 | HasBaseReg, Scale); | |||
1747 | } | |||
1748 | ||||
1749 | static bool isAMCompletelyFolded(const TargetTransformInfo &TTI, | |||
1750 | int64_t MinOffset, int64_t MaxOffset, | |||
1751 | LSRUse::KindType Kind, MemAccessTy AccessTy, | |||
1752 | const Formula &F, const Loop &L) { | |||
1753 | // For the purpose of isAMCompletelyFolded either having a canonical formula | |||
1754 | // or a scale not equal to zero is correct. | |||
1755 | // Problems may arise from non canonical formulae having a scale == 0. | |||
1756 | // Strictly speaking it would best to just rely on canonical formulae. | |||
1757 | // However, when we generate the scaled formulae, we first check that the | |||
1758 | // scaling factor is profitable before computing the actual ScaledReg for | |||
1759 | // compile time sake. | |||
1760 | assert((F.isCanonical(L) || F.Scale != 0))(static_cast <bool> ((F.isCanonical(L) || F.Scale != 0) ) ? void (0) : __assert_fail ("(F.isCanonical(L) || F.Scale != 0)" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 1760, __extension__ __PRETTY_FUNCTION__)); | |||
1761 | return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, | |||
1762 | F.BaseGV, F.BaseOffset, F.HasBaseReg, F.Scale); | |||
1763 | } | |||
1764 | ||||
1765 | /// Test whether we know how to expand the current formula. | |||
1766 | static bool isLegalUse(const TargetTransformInfo &TTI, int64_t MinOffset, | |||
1767 | int64_t MaxOffset, LSRUse::KindType Kind, | |||
1768 | MemAccessTy AccessTy, GlobalValue *BaseGV, | |||
1769 | int64_t BaseOffset, bool HasBaseReg, int64_t Scale) { | |||
1770 | // We know how to expand completely foldable formulae. | |||
1771 | return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV, | |||
1772 | BaseOffset, HasBaseReg, Scale) || | |||
1773 | // Or formulae that use a base register produced by a sum of base | |||
1774 | // registers. | |||
1775 | (Scale == 1 && | |||
1776 | isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, | |||
1777 | BaseGV, BaseOffset, true, 0)); | |||
1778 | } | |||
1779 | ||||
1780 | static bool isLegalUse(const TargetTransformInfo &TTI, int64_t MinOffset, | |||
1781 | int64_t MaxOffset, LSRUse::KindType Kind, | |||
1782 | MemAccessTy AccessTy, const Formula &F) { | |||
1783 | return isLegalUse(TTI, MinOffset, MaxOffset, Kind, AccessTy, F.BaseGV, | |||
1784 | F.BaseOffset, F.HasBaseReg, F.Scale); | |||
1785 | } | |||
1786 | ||||
1787 | static bool isAMCompletelyFolded(const TargetTransformInfo &TTI, | |||
1788 | const LSRUse &LU, const Formula &F) { | |||
1789 | // Target may want to look at the user instructions. | |||
1790 | if (LU.Kind == LSRUse::Address && TTI.LSRWithInstrQueries()) { | |||
1791 | for (const LSRFixup &Fixup : LU.Fixups) | |||
1792 | if (!isAMCompletelyFolded(TTI, LSRUse::Address, LU.AccessTy, F.BaseGV, | |||
1793 | (F.BaseOffset + Fixup.Offset), F.HasBaseReg, | |||
1794 | F.Scale, Fixup.UserInst)) | |||
1795 | return false; | |||
1796 | return true; | |||
1797 | } | |||
1798 | ||||
1799 | return isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, | |||
1800 | LU.AccessTy, F.BaseGV, F.BaseOffset, F.HasBaseReg, | |||
1801 | F.Scale); | |||
1802 | } | |||
1803 | ||||
1804 | static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI, | |||
1805 | const LSRUse &LU, const Formula &F, | |||
1806 | const Loop &L) { | |||
1807 | if (!F.Scale) | |||
1808 | return 0; | |||
1809 | ||||
1810 | // If the use is not completely folded in that instruction, we will have to | |||
1811 | // pay an extra cost only for scale != 1. | |||
1812 | if (!isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, | |||
1813 | LU.AccessTy, F, L)) | |||
1814 | return F.Scale != 1; | |||
1815 | ||||
1816 | switch (LU.Kind) { | |||
1817 | case LSRUse::Address: { | |||
1818 | // Check the scaling factor cost with both the min and max offsets. | |||
1819 | InstructionCost ScaleCostMinOffset = TTI.getScalingFactorCost( | |||
1820 | LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MinOffset, F.HasBaseReg, | |||
1821 | F.Scale, LU.AccessTy.AddrSpace); | |||
1822 | InstructionCost ScaleCostMaxOffset = TTI.getScalingFactorCost( | |||
1823 | LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MaxOffset, F.HasBaseReg, | |||
1824 | F.Scale, LU.AccessTy.AddrSpace); | |||
1825 | ||||
1826 | assert(ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() &&(static_cast <bool> (ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() && "Legal addressing mode has an illegal cost!" ) ? void (0) : __assert_fail ("ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() && \"Legal addressing mode has an illegal cost!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 1827, __extension__ __PRETTY_FUNCTION__)) | |||
1827 | "Legal addressing mode has an illegal cost!")(static_cast <bool> (ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() && "Legal addressing mode has an illegal cost!" ) ? void (0) : __assert_fail ("ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() && \"Legal addressing mode has an illegal cost!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 1827, __extension__ __PRETTY_FUNCTION__)); | |||
1828 | return std::max(ScaleCostMinOffset, ScaleCostMaxOffset); | |||
1829 | } | |||
1830 | case LSRUse::ICmpZero: | |||
1831 | case LSRUse::Basic: | |||
1832 | case LSRUse::Special: | |||
1833 | // The use is completely folded, i.e., everything is folded into the | |||
1834 | // instruction. | |||
1835 | return 0; | |||
1836 | } | |||
1837 | ||||
1838 | llvm_unreachable("Invalid LSRUse Kind!")::llvm::llvm_unreachable_internal("Invalid LSRUse Kind!", "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp" , 1838); | |||
1839 | } | |||
1840 | ||||
1841 | static bool isAlwaysFoldable(const TargetTransformInfo &TTI, | |||
1842 | LSRUse::KindType Kind, MemAccessTy AccessTy, | |||
1843 | GlobalValue *BaseGV, int64_t BaseOffset, | |||
1844 | bool HasBaseReg) { | |||
1845 | // Fast-path: zero is always foldable. | |||
1846 | if (BaseOffset == 0 && !BaseGV) return true; | |||
1847 | ||||
1848 | // Conservatively, create an address with an immediate and a | |||
1849 | // base and a scale. | |||
1850 | int64_t Scale = Kind == LSRUse::ICmpZero ? -1 : 1; | |||
1851 | ||||
1852 | // Canonicalize a scale of 1 to a base register if the formula doesn't | |||
1853 | // already have a base register. | |||
1854 | if (!HasBaseReg && Scale == 1) { | |||
1855 | Scale = 0; | |||
1856 | HasBaseReg = true; | |||
1857 | } | |||
1858 | ||||
1859 | return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset, | |||
1860 | HasBaseReg, Scale); | |||
1861 | } | |||
1862 | ||||
1863 | static bool isAlwaysFoldable(const TargetTransformInfo &TTI, | |||
1864 | ScalarEvolution &SE, int64_t MinOffset, | |||
1865 | int64_t MaxOffset, LSRUse::KindType Kind, | |||
1866 | MemAccessTy AccessTy, const SCEV *S, | |||
1867 | bool HasBaseReg) { | |||
1868 | // Fast-path: zero is always foldable. | |||
1869 | if (S->isZero()) return true; | |||
1870 | ||||
1871 | // Conservatively, create an address with an immediate and a | |||
1872 | // base and a scale. | |||
1873 | int64_t BaseOffset = ExtractImmediate(S, SE); | |||
1874 | GlobalValue *BaseGV = ExtractSymbol(S, SE); | |||
1875 | ||||
1876 | // If there's anything else involved, it's not foldable. | |||
1877 | if (!S->isZero()) return false; | |||
1878 | ||||
1879 | // Fast-path: zero is always foldable. | |||
1880 | if (BaseOffset == 0 && !BaseGV) return true; | |||
1881 | ||||
1882 | // Conservatively, create an address with an immediate and a | |||
1883 | // base and a scale. | |||
1884 | int64_t Scale = Kind == LSRUse::ICmpZero ? -1 : 1; | |||
1885 | ||||
1886 | return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV, | |||
1887 | BaseOffset, HasBaseReg, Scale); | |||
1888 | } | |||
1889 | ||||
1890 | namespace { | |||
1891 | ||||
1892 | /// An individual increment in a Chain of IV increments. Relate an IV user to | |||
1893 | /// an expression that computes the IV it uses from the IV used by the previous | |||
1894 | /// link in the Chain. | |||
1895 | /// | |||
1896 | /// For the head of a chain, IncExpr holds the absolute SCEV expression for the | |||
1897 | /// original IVOperand. The head of the chain's IVOperand is only valid during | |||
1898 | /// chain collection, before LSR replaces IV users. During chain generation, | |||
1899 | /// IncExpr can be used to find the new IVOperand that computes the same | |||
1900 | /// expression. | |||
1901 | struct IVInc { | |||
1902 | Instruction *UserInst; | |||
1903 | Value* IVOperand; | |||
1904 | const SCEV *IncExpr; | |||
1905 | ||||
1906 | IVInc(Instruction *U, Value *O, const SCEV *E) | |||
1907 | : UserInst(U), IVOperand(O), IncExpr(E) {} | |||
1908 | }; | |||
1909 | ||||
1910 | // The list of IV increments in program order. We typically add the head of a | |||
1911 | // chain without finding subsequent links. | |||
1912 | struct IVChain { | |||
1913 | SmallVector<IVInc, 1> Incs; | |||
1914 | const SCEV *ExprBase = nullptr; | |||
1915 | ||||
1916 | IVChain() = default; | |||
1917 | IVChain(const IVInc &Head, const SCEV *Base) | |||
1918 | : Incs(1, Head), ExprBase(Base) {} | |||
1919 | ||||
1920 | using const_iterator = SmallVectorImpl<IVInc>::const_iterator; | |||
1921 | ||||
1922 | // Return the first increment in the chain. | |||
1923 | const_iterator begin() const { | |||
1924 | assert(!Incs.empty())(static_cast <bool> (!Incs.empty()) ? void (0) : __assert_fail ("!Incs.empty()", "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp" , 1924, __extension__ __PRETTY_FUNCTION__)); | |||
1925 | return std::next(Incs.begin()); | |||
1926 | } | |||
1927 | const_iterator end() const { | |||
1928 | return Incs.end(); | |||
1929 | } | |||
1930 | ||||
1931 | // Returns true if this chain contains any increments. | |||
1932 | bool hasIncs() const { return Incs.size() >= 2; } | |||
1933 | ||||
1934 | // Add an IVInc to the end of this chain. | |||
1935 | void add(const IVInc &X) { Incs.push_back(X); } | |||
1936 | ||||
1937 | // Returns the last UserInst in the chain. | |||
1938 | Instruction *tailUserInst() const { return Incs.back().UserInst; } | |||
1939 | ||||
1940 | // Returns true if IncExpr can be profitably added to this chain. | |||
1941 | bool isProfitableIncrement(const SCEV *OperExpr, | |||
1942 | const SCEV *IncExpr, | |||
1943 | ScalarEvolution&); | |||
1944 | }; | |||
1945 | ||||
1946 | /// Helper for CollectChains to track multiple IV increment uses. Distinguish | |||
1947 | /// between FarUsers that definitely cross IV increments and NearUsers that may | |||
1948 | /// be used between IV increments. | |||
1949 | struct ChainUsers { | |||
1950 | SmallPtrSet<Instruction*, 4> FarUsers; | |||
1951 | SmallPtrSet<Instruction*, 4> NearUsers; | |||
1952 | }; | |||
1953 | ||||
1954 | /// This class holds state for the main loop strength reduction logic. | |||
1955 | class LSRInstance { | |||
1956 | IVUsers &IU; | |||
1957 | ScalarEvolution &SE; | |||
1958 | DominatorTree &DT; | |||
1959 | LoopInfo &LI; | |||
1960 | AssumptionCache &AC; | |||
1961 | TargetLibraryInfo &TLI; | |||
1962 | const TargetTransformInfo &TTI; | |||
1963 | Loop *const L; | |||
1964 | MemorySSAUpdater *MSSAU; | |||
1965 | TTI::AddressingModeKind AMK; | |||
1966 | mutable SCEVExpander Rewriter; | |||
1967 | bool Changed = false; | |||
1968 | ||||
1969 | /// This is the insert position that the current loop's induction variable | |||
1970 | /// increment should be placed. In simple loops, this is the latch block's | |||
1971 | /// terminator. But in more complicated cases, this is a position which will | |||
1972 | /// dominate all the in-loop post-increment users. | |||
1973 | Instruction *IVIncInsertPos = nullptr; | |||
1974 | ||||
1975 | /// Interesting factors between use strides. | |||
1976 | /// | |||
1977 | /// We explicitly use a SetVector which contains a SmallSet, instead of the | |||
1978 | /// default, a SmallDenseSet, because we need to use the full range of | |||
1979 | /// int64_ts, and there's currently no good way of doing that with | |||
1980 | /// SmallDenseSet. | |||
1981 | SetVector<int64_t, SmallVector<int64_t, 8>, SmallSet<int64_t, 8>> Factors; | |||
1982 | ||||
1983 | /// The cost of the current SCEV, the best solution by LSR will be dropped if | |||
1984 | /// the solution is not profitable. | |||
1985 | Cost BaselineCost; | |||
1986 | ||||
1987 | /// Interesting use types, to facilitate truncation reuse. | |||
1988 | SmallSetVector<Type *, 4> Types; | |||
1989 | ||||
1990 | /// The list of interesting uses. | |||
1991 | mutable SmallVector<LSRUse, 16> Uses; | |||
1992 | ||||
1993 | /// Track which uses use which register candidates. | |||
1994 | RegUseTracker RegUses; | |||
1995 | ||||
1996 | // Limit the number of chains to avoid quadratic behavior. We don't expect to | |||
1997 | // have more than a few IV increment chains in a loop. Missing a Chain falls | |||
1998 | // back to normal LSR behavior for those uses. | |||
1999 | static const unsigned MaxChains = 8; | |||
2000 | ||||
2001 | /// IV users can form a chain of IV increments. | |||
2002 | SmallVector<IVChain, MaxChains> IVChainVec; | |||
2003 | ||||
2004 | /// IV users that belong to profitable IVChains. | |||
2005 | SmallPtrSet<Use*, MaxChains> IVIncSet; | |||
2006 | ||||
2007 | /// Induction variables that were generated and inserted by the SCEV Expander. | |||
2008 | SmallVector<llvm::WeakVH, 2> ScalarEvolutionIVs; | |||
2009 | ||||
2010 | void OptimizeShadowIV(); | |||
2011 | bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse); | |||
2012 | ICmpInst *OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse); | |||
2013 | void OptimizeLoopTermCond(); | |||
2014 | ||||
2015 | void ChainInstruction(Instruction *UserInst, Instruction *IVOper, | |||
2016 | SmallVectorImpl<ChainUsers> &ChainUsersVec); | |||
2017 | void FinalizeChain(IVChain &Chain); | |||
2018 | void CollectChains(); | |||
2019 | void GenerateIVChain(const IVChain &Chain, | |||
2020 | SmallVectorImpl<WeakTrackingVH> &DeadInsts); | |||
2021 | ||||
2022 | void CollectInterestingTypesAndFactors(); | |||
2023 | void CollectFixupsAndInitialFormulae(); | |||
2024 | ||||
2025 | // Support for sharing of LSRUses between LSRFixups. | |||
2026 | using UseMapTy = DenseMap<LSRUse::SCEVUseKindPair, size_t>; | |||
2027 | UseMapTy UseMap; | |||
2028 | ||||
2029 | bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg, | |||
2030 | LSRUse::KindType Kind, MemAccessTy AccessTy); | |||
2031 | ||||
2032 | std::pair<size_t, int64_t> getUse(const SCEV *&Expr, LSRUse::KindType Kind, | |||
2033 | MemAccessTy AccessTy); | |||
2034 | ||||
2035 | void DeleteUse(LSRUse &LU, size_t LUIdx); | |||
2036 | ||||
2037 | LSRUse *FindUseWithSimilarFormula(const Formula &F, const LSRUse &OrigLU); | |||
2038 | ||||
2039 | void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx); | |||
2040 | void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx); | |||
2041 | void CountRegisters(const Formula &F, size_t LUIdx); | |||
2042 | bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F); | |||
2043 | ||||
2044 | void CollectLoopInvariantFixupsAndFormulae(); | |||
2045 | ||||
2046 | void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base, | |||
2047 | unsigned Depth = 0); | |||
2048 | ||||
2049 | void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx, | |||
2050 | const Formula &Base, unsigned Depth, | |||
2051 | size_t Idx, bool IsScaledReg = false); | |||
2052 | void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base); | |||
2053 | void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx, | |||
2054 | const Formula &Base, size_t Idx, | |||
2055 | bool IsScaledReg = false); | |||
2056 | void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base); | |||
2057 | void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx, | |||
2058 | const Formula &Base, | |||
2059 | const SmallVectorImpl<int64_t> &Worklist, | |||
2060 | size_t Idx, bool IsScaledReg = false); | |||
2061 | void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base); | |||
2062 | void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base); | |||
2063 | void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base); | |||
2064 | void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base); | |||
2065 | void GenerateCrossUseConstantOffsets(); | |||
2066 | void GenerateAllReuseFormulae(); | |||
2067 | ||||
2068 | void FilterOutUndesirableDedicatedRegisters(); | |||
2069 | ||||
2070 | size_t EstimateSearchSpaceComplexity() const; | |||
2071 | void NarrowSearchSpaceByDetectingSupersets(); | |||
2072 | void NarrowSearchSpaceByCollapsingUnrolledCode(); | |||
2073 | void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(); | |||
2074 | void NarrowSearchSpaceByFilterFormulaWithSameScaledReg(); | |||
2075 | void NarrowSearchSpaceByFilterPostInc(); | |||
2076 | void NarrowSearchSpaceByDeletingCostlyFormulas(); | |||
2077 | void NarrowSearchSpaceByPickingWinnerRegs(); | |||
2078 | void NarrowSearchSpaceUsingHeuristics(); | |||
2079 | ||||
2080 | void SolveRecurse(SmallVectorImpl<const Formula *> &Solution, | |||
2081 | Cost &SolutionCost, | |||
2082 | SmallVectorImpl<const Formula *> &Workspace, | |||
2083 | const Cost &CurCost, | |||
2084 | const SmallPtrSet<const SCEV *, 16> &CurRegs, | |||
2085 | DenseSet<const SCEV *> &VisitedRegs) const; | |||
2086 | void Solve(SmallVectorImpl<const Formula *> &Solution) const; | |||
2087 | ||||
2088 | BasicBlock::iterator | |||
2089 | HoistInsertPosition(BasicBlock::iterator IP, | |||
2090 | const SmallVectorImpl<Instruction *> &Inputs) const; | |||
2091 | BasicBlock::iterator AdjustInsertPositionForExpand(BasicBlock::iterator IP, | |||
2092 | const LSRFixup &LF, | |||
2093 | const LSRUse &LU) const; | |||
2094 | ||||
2095 | Value *Expand(const LSRUse &LU, const LSRFixup &LF, const Formula &F, | |||
2096 | BasicBlock::iterator IP, | |||
2097 | SmallVectorImpl<WeakTrackingVH> &DeadInsts) const; | |||
2098 | void RewriteForPHI(PHINode *PN, const LSRUse &LU, const LSRFixup &LF, | |||
2099 | const Formula &F, | |||
2100 | SmallVectorImpl<WeakTrackingVH> &DeadInsts) const; | |||
2101 | void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F, | |||
2102 | SmallVectorImpl<WeakTrackingVH> &DeadInsts) const; | |||
2103 | void ImplementSolution(const SmallVectorImpl<const Formula *> &Solution); | |||
2104 | ||||
2105 | public: | |||
2106 | LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT, | |||
2107 | LoopInfo &LI, const TargetTransformInfo &TTI, AssumptionCache &AC, | |||
2108 | TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU); | |||
2109 | ||||
2110 | bool getChanged() const { return Changed; } | |||
2111 | const SmallVectorImpl<WeakVH> &getScalarEvolutionIVs() const { | |||
2112 | return ScalarEvolutionIVs; | |||
2113 | } | |||
2114 | ||||
2115 | void print_factors_and_types(raw_ostream &OS) const; | |||
2116 | void print_fixups(raw_ostream &OS) const; | |||
2117 | void print_uses(raw_ostream &OS) const; | |||
2118 | void print(raw_ostream &OS) const; | |||
2119 | void dump() const; | |||
2120 | }; | |||
2121 | ||||
2122 | } // end anonymous namespace | |||
2123 | ||||
2124 | /// If IV is used in a int-to-float cast inside the loop then try to eliminate | |||
2125 | /// the cast operation. | |||
2126 | void LSRInstance::OptimizeShadowIV() { | |||
2127 | const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); | |||
2128 | if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) | |||
2129 | return; | |||
2130 | ||||
2131 | for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); | |||
2132 | UI != E; /* empty */) { | |||
2133 | IVUsers::const_iterator CandidateUI = UI; | |||
2134 | ++UI; | |||
2135 | Instruction *ShadowUse = CandidateUI->getUser(); | |||
2136 | Type *DestTy = nullptr; | |||
2137 | bool IsSigned = false; | |||
2138 | ||||
2139 | /* If shadow use is a int->float cast then insert a second IV | |||
2140 | to eliminate this cast. | |||
2141 | ||||
2142 | for (unsigned i = 0; i < n; ++i) | |||
2143 | foo((double)i); | |||
2144 | ||||
2145 | is transformed into | |||
2146 | ||||
2147 | double d = 0.0; | |||
2148 | for (unsigned i = 0; i < n; ++i, ++d) | |||
2149 | foo(d); | |||
2150 | */ | |||
2151 | if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser())) { | |||
2152 | IsSigned = false; | |||
2153 | DestTy = UCast->getDestTy(); | |||
2154 | } | |||
2155 | else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser())) { | |||
2156 | IsSigned = true; | |||
2157 | DestTy = SCast->getDestTy(); | |||
2158 | } | |||
2159 | if (!DestTy) continue; | |||
2160 | ||||
2161 | // If target does not support DestTy natively then do not apply | |||
2162 | // this transformation. | |||
2163 | if (!TTI.isTypeLegal(DestTy)) continue; | |||
2164 | ||||
2165 | PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0)); | |||
2166 | if (!PH) continue; | |||
2167 | if (PH->getNumIncomingValues() != 2) continue; | |||
2168 | ||||
2169 | // If the calculation in integers overflows, the result in FP type will | |||
2170 | // differ. So we only can do this transformation if we are guaranteed to not | |||
2171 | // deal with overflowing values | |||
2172 | const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(PH)); | |||
2173 | if (!AR) continue; | |||
2174 | if (IsSigned && !AR->hasNoSignedWrap()) continue; | |||
2175 | if (!IsSigned && !AR->hasNoUnsignedWrap()) continue; | |||
2176 | ||||
2177 | Type *SrcTy = PH->getType(); | |||
2178 | int Mantissa = DestTy->getFPMantissaWidth(); | |||
2179 | if (Mantissa == -1) continue; | |||
2180 | if ((int)SE.getTypeSizeInBits(SrcTy) > Mantissa) | |||
2181 | continue; | |||
2182 | ||||
2183 | unsigned Entry, Latch; | |||
2184 | if (PH->getIncomingBlock(0) == L->getLoopPreheader()) { | |||
2185 | Entry = 0; | |||
2186 | Latch = 1; | |||
2187 | } else { | |||
2188 | Entry = 1; | |||
2189 | Latch = 0; | |||
2190 | } | |||
2191 | ||||
2192 | ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry)); | |||
2193 | if (!Init) continue; | |||
2194 | Constant *NewInit = ConstantFP::get(DestTy, IsSigned ? | |||
2195 | (double)Init->getSExtValue() : | |||
2196 | (double)Init->getZExtValue()); | |||
2197 | ||||
2198 | BinaryOperator *Incr = | |||
2199 | dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch)); | |||
2200 | if (!Incr) continue; | |||
2201 | if (Incr->getOpcode() != Instruction::Add | |||
2202 | && Incr->getOpcode() != Instruction::Sub) | |||
2203 | continue; | |||
2204 | ||||
2205 | /* Initialize new IV, double d = 0.0 in above example. */ | |||
2206 | ConstantInt *C = nullptr; | |||
2207 | if (Incr->getOperand(0) == PH) | |||
2208 | C = dyn_cast<ConstantInt>(Incr->getOperand(1)); | |||
2209 | else if (Incr->getOperand(1) == PH) | |||
2210 | C = dyn_cast<ConstantInt>(Incr->getOperand(0)); | |||
2211 | else | |||
2212 | continue; | |||
2213 | ||||
2214 | if (!C) continue; | |||
2215 | ||||
2216 | // Ignore negative constants, as the code below doesn't handle them | |||
2217 | // correctly. TODO: Remove this restriction. | |||
2218 | if (!C->getValue().isStrictlyPositive()) continue; | |||
2219 | ||||
2220 | /* Add new PHINode. */ | |||
2221 | PHINode *NewPH = PHINode::Create(DestTy, 2, "IV.S.", PH); | |||
2222 | ||||
2223 | /* create new increment. '++d' in above example. */ | |||
2224 | Constant *CFP = ConstantFP::get(DestTy, C->getZExtValue()); | |||
2225 | BinaryOperator *NewIncr = | |||
2226 | BinaryOperator::Create(Incr->getOpcode() == Instruction::Add ? | |||
2227 | Instruction::FAdd : Instruction::FSub, | |||
2228 | NewPH, CFP, "IV.S.next.", Incr); | |||
2229 | ||||
2230 | NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry)); | |||
2231 | NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch)); | |||
2232 | ||||
2233 | /* Remove cast operation */ | |||
2234 | ShadowUse->replaceAllUsesWith(NewPH); | |||
2235 | ShadowUse->eraseFromParent(); | |||
2236 | Changed = true; | |||
2237 | break; | |||
2238 | } | |||
2239 | } | |||
2240 | ||||
2241 | /// If Cond has an operand that is an expression of an IV, set the IV user and | |||
2242 | /// stride information and return true, otherwise return false. | |||
2243 | bool LSRInstance::FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse) { | |||
2244 | for (IVStrideUse &U : IU) | |||
2245 | if (U.getUser() == Cond) { | |||
2246 | // NOTE: we could handle setcc instructions with multiple uses here, but | |||
2247 | // InstCombine does it as well for simple uses, it's not clear that it | |||
2248 | // occurs enough in real life to handle. | |||
2249 | CondUse = &U; | |||
2250 | return true; | |||
2251 | } | |||
2252 | return false; | |||
2253 | } | |||
2254 | ||||
2255 | /// Rewrite the loop's terminating condition if it uses a max computation. | |||
2256 | /// | |||
2257 | /// This is a narrow solution to a specific, but acute, problem. For loops | |||
2258 | /// like this: | |||
2259 | /// | |||
2260 | /// i = 0; | |||
2261 | /// do { | |||
2262 | /// p[i] = 0.0; | |||
2263 | /// } while (++i < n); | |||
2264 | /// | |||
2265 | /// the trip count isn't just 'n', because 'n' might not be positive. And | |||
2266 | /// unfortunately this can come up even for loops where the user didn't use | |||
2267 | /// a C do-while loop. For example, seemingly well-behaved top-test loops | |||
2268 | /// will commonly be lowered like this: | |||
2269 | /// | |||
2270 | /// if (n > 0) { | |||
2271 | /// i = 0; | |||
2272 | /// do { | |||
2273 | /// p[i] = 0.0; | |||
2274 | /// } while (++i < n); | |||
2275 | /// } | |||
2276 | /// | |||
2277 | /// and then it's possible for subsequent optimization to obscure the if | |||
2278 | /// test in such a way that indvars can't find it. | |||
2279 | /// | |||
2280 | /// When indvars can't find the if test in loops like this, it creates a | |||
2281 | /// max expression, which allows it to give the loop a canonical | |||
2282 | /// induction variable: | |||
2283 | /// | |||
2284 | /// i = 0; | |||
2285 | /// max = n < 1 ? 1 : n; | |||
2286 | /// do { | |||
2287 | /// p[i] = 0.0; | |||
2288 | /// } while (++i != max); | |||
2289 | /// | |||
2290 | /// Canonical induction variables are necessary because the loop passes | |||
2291 | /// are designed around them. The most obvious example of this is the | |||
2292 | /// LoopInfo analysis, which doesn't remember trip count values. It | |||
2293 | /// expects to be able to rediscover the trip count each time it is | |||
2294 | /// needed, and it does this using a simple analysis that only succeeds if | |||
2295 | /// the loop has a canonical induction variable. | |||
2296 | /// | |||
2297 | /// However, when it comes time to generate code, the maximum operation | |||
2298 | /// can be quite costly, especially if it's inside of an outer loop. | |||
2299 | /// | |||
2300 | /// This function solves this problem by detecting this type of loop and | |||
2301 | /// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting | |||
2302 | /// the instructions for the maximum computation. | |||
2303 | ICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) { | |||
2304 | // Check that the loop matches the pattern we're looking for. | |||
2305 | if (Cond->getPredicate() != CmpInst::ICMP_EQ && | |||
2306 | Cond->getPredicate() != CmpInst::ICMP_NE) | |||
2307 | return Cond; | |||
2308 | ||||
2309 | SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1)); | |||
2310 | if (!Sel || !Sel->hasOneUse()) return Cond; | |||
2311 | ||||
2312 | const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); | |||
2313 | if (isa<SCEVCouldNotCompute>(BackedgeTakenCount)) | |||
2314 | return Cond; | |||
2315 | const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1); | |||
2316 | ||||
2317 | // Add one to the backedge-taken count to get the trip count. | |||
2318 | const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount); | |||
2319 | if (IterationCount != SE.getSCEV(Sel)) return Cond; | |||
2320 | ||||
2321 | // Check for a max calculation that matches the pattern. There's no check | |||
2322 | // for ICMP_ULE here because the comparison would be with zero, which | |||
2323 | // isn't interesting. | |||
2324 | CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE; | |||
2325 | const SCEVNAryExpr *Max = nullptr; | |||
2326 | if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(BackedgeTakenCount)) { | |||
2327 | Pred = ICmpInst::ICMP_SLE; | |||
2328 | Max = S; | |||
2329 | } else if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(IterationCount)) { | |||
2330 | Pred = ICmpInst::ICMP_SLT; | |||
2331 | Max = S; | |||
2332 | } else if (const SCEVUMaxExpr *U = dyn_cast<SCEVUMaxExpr>(IterationCount)) { | |||
2333 | Pred = ICmpInst::ICMP_ULT; | |||
2334 | Max = U; | |||
2335 | } else { | |||
2336 | // No match; bail. | |||
2337 | return Cond; | |||
2338 | } | |||
2339 | ||||
2340 | // To handle a max with more than two operands, this optimization would | |||
2341 | // require additional checking and setup. | |||
2342 | if (Max->getNumOperands() != 2) | |||
2343 | return Cond; | |||
2344 | ||||
2345 | const SCEV *MaxLHS = Max->getOperand(0); | |||
2346 | const SCEV *MaxRHS = Max->getOperand(1); | |||
2347 | ||||
2348 | // ScalarEvolution canonicalizes constants to the left. For < and >, look | |||
2349 | // for a comparison with 1. For <= and >=, a comparison with zero. | |||
2350 | if (!MaxLHS || | |||
2351 | (ICmpInst::isTrueWhenEqual(Pred) ? !MaxLHS->isZero() : (MaxLHS != One))) | |||
2352 | return Cond; | |||
2353 | ||||
2354 | // Check the relevant induction variable for conformance to | |||
2355 | // the pattern. | |||
2356 | const SCEV *IV = SE.getSCEV(Cond->getOperand(0)); | |||
2357 | const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV); | |||
2358 | if (!AR || !AR->isAffine() || | |||
2359 | AR->getStart() != One || | |||
2360 | AR->getStepRecurrence(SE) != One) | |||
2361 | return Cond; | |||
2362 | ||||
2363 | assert(AR->getLoop() == L &&(static_cast <bool> (AR->getLoop() == L && "Loop condition operand is an addrec in a different loop!" ) ? void (0) : __assert_fail ("AR->getLoop() == L && \"Loop condition operand is an addrec in a different loop!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 2364, __extension__ __PRETTY_FUNCTION__)) | |||
2364 | "Loop condition operand is an addrec in a different loop!")(static_cast <bool> (AR->getLoop() == L && "Loop condition operand is an addrec in a different loop!" ) ? void (0) : __assert_fail ("AR->getLoop() == L && \"Loop condition operand is an addrec in a different loop!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 2364, __extension__ __PRETTY_FUNCTION__)); | |||
2365 | ||||
2366 | // Check the right operand of the select, and remember it, as it will | |||
2367 | // be used in the new comparison instruction. | |||
2368 | Value *NewRHS = nullptr; | |||
2369 | if (ICmpInst::isTrueWhenEqual(Pred)) { | |||
2370 | // Look for n+1, and grab n. | |||
2371 | if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(1))) | |||
2372 | if (ConstantInt *BO1 = dyn_cast<ConstantInt>(BO->getOperand(1))) | |||
2373 | if (BO1->isOne() && SE.getSCEV(BO->getOperand(0)) == MaxRHS) | |||
2374 | NewRHS = BO->getOperand(0); | |||
2375 | if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(2))) | |||
2376 | if (ConstantInt *BO1 = dyn_cast<ConstantInt>(BO->getOperand(1))) | |||
2377 | if (BO1->isOne() && SE.getSCEV(BO->getOperand(0)) == MaxRHS) | |||
2378 | NewRHS = BO->getOperand(0); | |||
2379 | if (!NewRHS) | |||
2380 | return Cond; | |||
2381 | } else if (SE.getSCEV(Sel->getOperand(1)) == MaxRHS) | |||
2382 | NewRHS = Sel->getOperand(1); | |||
2383 | else if (SE.getSCEV(Sel->getOperand(2)) == MaxRHS) | |||
2384 | NewRHS = Sel->getOperand(2); | |||
2385 | else if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(MaxRHS)) | |||
2386 | NewRHS = SU->getValue(); | |||
2387 | else | |||
2388 | // Max doesn't match expected pattern. | |||
2389 | return Cond; | |||
2390 | ||||
2391 | // Determine the new comparison opcode. It may be signed or unsigned, | |||
2392 | // and the original comparison may be either equality or inequality. | |||
2393 | if (Cond->getPredicate() == CmpInst::ICMP_EQ) | |||
2394 | Pred = CmpInst::getInversePredicate(Pred); | |||
2395 | ||||
2396 | // Ok, everything looks ok to change the condition into an SLT or SGE and | |||
2397 | // delete the max calculation. | |||
2398 | ICmpInst *NewCond = | |||
2399 | new ICmpInst(Cond, Pred, Cond->getOperand(0), NewRHS, "scmp"); | |||
2400 | ||||
2401 | // Delete the max calculation instructions. | |||
2402 | NewCond->setDebugLoc(Cond->getDebugLoc()); | |||
2403 | Cond->replaceAllUsesWith(NewCond); | |||
2404 | CondUse->setUser(NewCond); | |||
2405 | Instruction *Cmp = cast<Instruction>(Sel->getOperand(0)); | |||
2406 | Cond->eraseFromParent(); | |||
2407 | Sel->eraseFromParent(); | |||
2408 | if (Cmp->use_empty()) | |||
2409 | Cmp->eraseFromParent(); | |||
2410 | return NewCond; | |||
2411 | } | |||
2412 | ||||
2413 | /// Change loop terminating condition to use the postinc iv when possible. | |||
2414 | void | |||
2415 | LSRInstance::OptimizeLoopTermCond() { | |||
2416 | SmallPtrSet<Instruction *, 4> PostIncs; | |||
2417 | ||||
2418 | // We need a different set of heuristics for rotated and non-rotated loops. | |||
2419 | // If a loop is rotated then the latch is also the backedge, so inserting | |||
2420 | // post-inc expressions just before the latch is ideal. To reduce live ranges | |||
2421 | // it also makes sense to rewrite terminating conditions to use post-inc | |||
2422 | // expressions. | |||
2423 | // | |||
2424 | // If the loop is not rotated then the latch is not a backedge; the latch | |||
2425 | // check is done in the loop head. Adding post-inc expressions before the | |||
2426 | // latch will cause overlapping live-ranges of pre-inc and post-inc expressions | |||
2427 | // in the loop body. In this case we do *not* want to use post-inc expressions | |||
2428 | // in the latch check, and we want to insert post-inc expressions before | |||
2429 | // the backedge. | |||
2430 | BasicBlock *LatchBlock = L->getLoopLatch(); | |||
2431 | SmallVector<BasicBlock*, 8> ExitingBlocks; | |||
2432 | L->getExitingBlocks(ExitingBlocks); | |||
2433 | if (!llvm::is_contained(ExitingBlocks, LatchBlock)) { | |||
2434 | // The backedge doesn't exit the loop; treat this as a head-tested loop. | |||
2435 | IVIncInsertPos = LatchBlock->getTerminator(); | |||
2436 | return; | |||
2437 | } | |||
2438 | ||||
2439 | // Otherwise treat this as a rotated loop. | |||
2440 | for (BasicBlock *ExitingBlock : ExitingBlocks) { | |||
2441 | // Get the terminating condition for the loop if possible. If we | |||
2442 | // can, we want to change it to use a post-incremented version of its | |||
2443 | // induction variable, to allow coalescing the live ranges for the IV into | |||
2444 | // one register value. | |||
2445 | ||||
2446 | BranchInst *TermBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator()); | |||
2447 | if (!TermBr) | |||
2448 | continue; | |||
2449 | // FIXME: Overly conservative, termination condition could be an 'or' etc.. | |||
2450 | if (TermBr->isUnconditional() || !isa<ICmpInst>(TermBr->getCondition())) | |||
2451 | continue; | |||
2452 | ||||
2453 | // Search IVUsesByStride to find Cond's IVUse if there is one. | |||
2454 | IVStrideUse *CondUse = nullptr; | |||
2455 | ICmpInst *Cond = cast<ICmpInst>(TermBr->getCondition()); | |||
2456 | if (!FindIVUserForCond(Cond, CondUse)) | |||
2457 | continue; | |||
2458 | ||||
2459 | // If the trip count is computed in terms of a max (due to ScalarEvolution | |||
2460 | // being unable to find a sufficient guard, for example), change the loop | |||
2461 | // comparison to use SLT or ULT instead of NE. | |||
2462 | // One consequence of doing this now is that it disrupts the count-down | |||
2463 | // optimization. That's not always a bad thing though, because in such | |||
2464 | // cases it may still be worthwhile to avoid a max. | |||
2465 | Cond = OptimizeMax(Cond, CondUse); | |||
2466 | ||||
2467 | // If this exiting block dominates the latch block, it may also use | |||
2468 | // the post-inc value if it won't be shared with other uses. | |||
2469 | // Check for dominance. | |||
2470 | if (!DT.dominates(ExitingBlock, LatchBlock)) | |||
2471 | continue; | |||
2472 | ||||
2473 | // Conservatively avoid trying to use the post-inc value in non-latch | |||
2474 | // exits if there may be pre-inc users in intervening blocks. | |||
2475 | if (LatchBlock != ExitingBlock) | |||
2476 | for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) | |||
2477 | // Test if the use is reachable from the exiting block. This dominator | |||
2478 | // query is a conservative approximation of reachability. | |||
2479 | if (&*UI != CondUse && | |||
2480 | !DT.properlyDominates(UI->getUser()->getParent(), ExitingBlock)) { | |||
2481 | // Conservatively assume there may be reuse if the quotient of their | |||
2482 | // strides could be a legal scale. | |||
2483 | const SCEV *A = IU.getStride(*CondUse, L); | |||
2484 | const SCEV *B = IU.getStride(*UI, L); | |||
2485 | if (!A || !B) continue; | |||
2486 | if (SE.getTypeSizeInBits(A->getType()) != | |||
2487 | SE.getTypeSizeInBits(B->getType())) { | |||
2488 | if (SE.getTypeSizeInBits(A->getType()) > | |||
2489 | SE.getTypeSizeInBits(B->getType())) | |||
2490 | B = SE.getSignExtendExpr(B, A->getType()); | |||
2491 | else | |||
2492 | A = SE.getSignExtendExpr(A, B->getType()); | |||
2493 | } | |||
2494 | if (const SCEVConstant *D = | |||
2495 | dyn_cast_or_null<SCEVConstant>(getExactSDiv(B, A, SE))) { | |||
2496 | const ConstantInt *C = D->getValue(); | |||
2497 | // Stride of one or negative one can have reuse with non-addresses. | |||
2498 | if (C->isOne() || C->isMinusOne()) | |||
2499 | goto decline_post_inc; | |||
2500 | // Avoid weird situations. | |||
2501 | if (C->getValue().getMinSignedBits() >= 64 || | |||
2502 | C->getValue().isMinSignedValue()) | |||
2503 | goto decline_post_inc; | |||
2504 | // Check for possible scaled-address reuse. | |||
2505 | if (isAddressUse(TTI, UI->getUser(), UI->getOperandValToReplace())) { | |||
2506 | MemAccessTy AccessTy = getAccessType( | |||
2507 | TTI, UI->getUser(), UI->getOperandValToReplace()); | |||
2508 | int64_t Scale = C->getSExtValue(); | |||
2509 | if (TTI.isLegalAddressingMode(AccessTy.MemTy, /*BaseGV=*/nullptr, | |||
2510 | /*BaseOffset=*/0, | |||
2511 | /*HasBaseReg=*/false, Scale, | |||
2512 | AccessTy.AddrSpace)) | |||
2513 | goto decline_post_inc; | |||
2514 | Scale = -Scale; | |||
2515 | if (TTI.isLegalAddressingMode(AccessTy.MemTy, /*BaseGV=*/nullptr, | |||
2516 | /*BaseOffset=*/0, | |||
2517 | /*HasBaseReg=*/false, Scale, | |||
2518 | AccessTy.AddrSpace)) | |||
2519 | goto decline_post_inc; | |||
2520 | } | |||
2521 | } | |||
2522 | } | |||
2523 | ||||
2524 | LLVM_DEBUG(dbgs() << " Change loop exiting icmp to use postinc iv: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Change loop exiting icmp to use postinc iv: " << *Cond << '\n'; } } while (false) | |||
2525 | << *Cond << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Change loop exiting icmp to use postinc iv: " << *Cond << '\n'; } } while (false); | |||
2526 | ||||
2527 | // It's possible for the setcc instruction to be anywhere in the loop, and | |||
2528 | // possible for it to have multiple users. If it is not immediately before | |||
2529 | // the exiting block branch, move it. | |||
2530 | if (Cond->getNextNonDebugInstruction() != TermBr) { | |||
2531 | if (Cond->hasOneUse()) { | |||
2532 | Cond->moveBefore(TermBr); | |||
2533 | } else { | |||
2534 | // Clone the terminating condition and insert into the loopend. | |||
2535 | ICmpInst *OldCond = Cond; | |||
2536 | Cond = cast<ICmpInst>(Cond->clone()); | |||
2537 | Cond->setName(L->getHeader()->getName() + ".termcond"); | |||
2538 | Cond->insertInto(ExitingBlock, TermBr->getIterator()); | |||
2539 | ||||
2540 | // Clone the IVUse, as the old use still exists! | |||
2541 | CondUse = &IU.AddUser(Cond, CondUse->getOperandValToReplace()); | |||
2542 | TermBr->replaceUsesOfWith(OldCond, Cond); | |||
2543 | } | |||
2544 | } | |||
2545 | ||||
2546 | // If we get to here, we know that we can transform the setcc instruction to | |||
2547 | // use the post-incremented version of the IV, allowing us to coalesce the | |||
2548 | // live ranges for the IV correctly. | |||
2549 | CondUse->transformToPostInc(L); | |||
2550 | Changed = true; | |||
2551 | ||||
2552 | PostIncs.insert(Cond); | |||
2553 | decline_post_inc:; | |||
2554 | } | |||
2555 | ||||
2556 | // Determine an insertion point for the loop induction variable increment. It | |||
2557 | // must dominate all the post-inc comparisons we just set up, and it must | |||
2558 | // dominate the loop latch edge. | |||
2559 | IVIncInsertPos = L->getLoopLatch()->getTerminator(); | |||
2560 | for (Instruction *Inst : PostIncs) | |||
2561 | IVIncInsertPos = DT.findNearestCommonDominator(IVIncInsertPos, Inst); | |||
2562 | } | |||
2563 | ||||
2564 | /// Determine if the given use can accommodate a fixup at the given offset and | |||
2565 | /// other details. If so, update the use and return true. | |||
2566 | bool LSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset, | |||
2567 | bool HasBaseReg, LSRUse::KindType Kind, | |||
2568 | MemAccessTy AccessTy) { | |||
2569 | int64_t NewMinOffset = LU.MinOffset; | |||
2570 | int64_t NewMaxOffset = LU.MaxOffset; | |||
2571 | MemAccessTy NewAccessTy = AccessTy; | |||
2572 | ||||
2573 | // Check for a mismatched kind. It's tempting to collapse mismatched kinds to | |||
2574 | // something conservative, however this can pessimize in the case that one of | |||
2575 | // the uses will have all its uses outside the loop, for example. | |||
2576 | if (LU.Kind != Kind) | |||
2577 | return false; | |||
2578 | ||||
2579 | // Check for a mismatched access type, and fall back conservatively as needed. | |||
2580 | // TODO: Be less conservative when the type is similar and can use the same | |||
2581 | // addressing modes. | |||
2582 | if (Kind == LSRUse::Address) { | |||
2583 | if (AccessTy.MemTy != LU.AccessTy.MemTy) { | |||
2584 | NewAccessTy = MemAccessTy::getUnknown(AccessTy.MemTy->getContext(), | |||
2585 | AccessTy.AddrSpace); | |||
2586 | } | |||
2587 | } | |||
2588 | ||||
2589 | // Conservatively assume HasBaseReg is true for now. | |||
2590 | if (NewOffset < LU.MinOffset) { | |||
2591 | if (!isAlwaysFoldable(TTI, Kind, NewAccessTy, /*BaseGV=*/nullptr, | |||
2592 | LU.MaxOffset - NewOffset, HasBaseReg)) | |||
2593 | return false; | |||
2594 | NewMinOffset = NewOffset; | |||
2595 | } else if (NewOffset > LU.MaxOffset) { | |||
2596 | if (!isAlwaysFoldable(TTI, Kind, NewAccessTy, /*BaseGV=*/nullptr, | |||
2597 | NewOffset - LU.MinOffset, HasBaseReg)) | |||
2598 | return false; | |||
2599 | NewMaxOffset = NewOffset; | |||
2600 | } | |||
2601 | ||||
2602 | // Update the use. | |||
2603 | LU.MinOffset = NewMinOffset; | |||
2604 | LU.MaxOffset = NewMaxOffset; | |||
2605 | LU.AccessTy = NewAccessTy; | |||
2606 | return true; | |||
2607 | } | |||
2608 | ||||
2609 | /// Return an LSRUse index and an offset value for a fixup which needs the given | |||
2610 | /// expression, with the given kind and optional access type. Either reuse an | |||
2611 | /// existing use or create a new one, as needed. | |||
2612 | std::pair<size_t, int64_t> LSRInstance::getUse(const SCEV *&Expr, | |||
2613 | LSRUse::KindType Kind, | |||
2614 | MemAccessTy AccessTy) { | |||
2615 | const SCEV *Copy = Expr; | |||
2616 | int64_t Offset = ExtractImmediate(Expr, SE); | |||
2617 | ||||
2618 | // Basic uses can't accept any offset, for example. | |||
2619 | if (!isAlwaysFoldable(TTI, Kind, AccessTy, /*BaseGV=*/ nullptr, | |||
2620 | Offset, /*HasBaseReg=*/ true)) { | |||
2621 | Expr = Copy; | |||
2622 | Offset = 0; | |||
2623 | } | |||
2624 | ||||
2625 | std::pair<UseMapTy::iterator, bool> P = | |||
2626 | UseMap.insert(std::make_pair(LSRUse::SCEVUseKindPair(Expr, Kind), 0)); | |||
2627 | if (!P.second) { | |||
2628 | // A use already existed with this base. | |||
2629 | size_t LUIdx = P.first->second; | |||
2630 | LSRUse &LU = Uses[LUIdx]; | |||
2631 | if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy)) | |||
2632 | // Reuse this use. | |||
2633 | return std::make_pair(LUIdx, Offset); | |||
2634 | } | |||
2635 | ||||
2636 | // Create a new use. | |||
2637 | size_t LUIdx = Uses.size(); | |||
2638 | P.first->second = LUIdx; | |||
2639 | Uses.push_back(LSRUse(Kind, AccessTy)); | |||
2640 | LSRUse &LU = Uses[LUIdx]; | |||
2641 | ||||
2642 | LU.MinOffset = Offset; | |||
2643 | LU.MaxOffset = Offset; | |||
2644 | return std::make_pair(LUIdx, Offset); | |||
2645 | } | |||
2646 | ||||
2647 | /// Delete the given use from the Uses list. | |||
2648 | void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) { | |||
2649 | if (&LU != &Uses.back()) | |||
2650 | std::swap(LU, Uses.back()); | |||
2651 | Uses.pop_back(); | |||
2652 | ||||
2653 | // Update RegUses. | |||
2654 | RegUses.swapAndDropUse(LUIdx, Uses.size()); | |||
2655 | } | |||
2656 | ||||
2657 | /// Look for a use distinct from OrigLU which is has a formula that has the same | |||
2658 | /// registers as the given formula. | |||
2659 | LSRUse * | |||
2660 | LSRInstance::FindUseWithSimilarFormula(const Formula &OrigF, | |||
2661 | const LSRUse &OrigLU) { | |||
2662 | // Search all uses for the formula. This could be more clever. | |||
2663 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { | |||
2664 | LSRUse &LU = Uses[LUIdx]; | |||
2665 | // Check whether this use is close enough to OrigLU, to see whether it's | |||
2666 | // worthwhile looking through its formulae. | |||
2667 | // Ignore ICmpZero uses because they may contain formulae generated by | |||
2668 | // GenerateICmpZeroScales, in which case adding fixup offsets may | |||
2669 | // be invalid. | |||
2670 | if (&LU != &OrigLU && | |||
2671 | LU.Kind != LSRUse::ICmpZero && | |||
2672 | LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy && | |||
2673 | LU.WidestFixupType == OrigLU.WidestFixupType && | |||
2674 | LU.HasFormulaWithSameRegs(OrigF)) { | |||
2675 | // Scan through this use's formulae. | |||
2676 | for (const Formula &F : LU.Formulae) { | |||
2677 | // Check to see if this formula has the same registers and symbols | |||
2678 | // as OrigF. | |||
2679 | if (F.BaseRegs == OrigF.BaseRegs && | |||
2680 | F.ScaledReg == OrigF.ScaledReg && | |||
2681 | F.BaseGV == OrigF.BaseGV && | |||
2682 | F.Scale == OrigF.Scale && | |||
2683 | F.UnfoldedOffset == OrigF.UnfoldedOffset) { | |||
2684 | if (F.BaseOffset == 0) | |||
2685 | return &LU; | |||
2686 | // This is the formula where all the registers and symbols matched; | |||
2687 | // there aren't going to be any others. Since we declined it, we | |||
2688 | // can skip the rest of the formulae and proceed to the next LSRUse. | |||
2689 | break; | |||
2690 | } | |||
2691 | } | |||
2692 | } | |||
2693 | } | |||
2694 | ||||
2695 | // Nothing looked good. | |||
2696 | return nullptr; | |||
2697 | } | |||
2698 | ||||
2699 | void LSRInstance::CollectInterestingTypesAndFactors() { | |||
2700 | SmallSetVector<const SCEV *, 4> Strides; | |||
2701 | ||||
2702 | // Collect interesting types and strides. | |||
2703 | SmallVector<const SCEV *, 4> Worklist; | |||
2704 | for (const IVStrideUse &U : IU) { | |||
2705 | const SCEV *Expr = IU.getExpr(U); | |||
2706 | ||||
2707 | // Collect interesting types. | |||
2708 | Types.insert(SE.getEffectiveSCEVType(Expr->getType())); | |||
2709 | ||||
2710 | // Add strides for mentioned loops. | |||
2711 | Worklist.push_back(Expr); | |||
2712 | do { | |||
2713 | const SCEV *S = Worklist.pop_back_val(); | |||
2714 | if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { | |||
2715 | if (AR->getLoop() == L) | |||
2716 | Strides.insert(AR->getStepRecurrence(SE)); | |||
2717 | Worklist.push_back(AR->getStart()); | |||
2718 | } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { | |||
2719 | append_range(Worklist, Add->operands()); | |||
2720 | } | |||
2721 | } while (!Worklist.empty()); | |||
2722 | } | |||
2723 | ||||
2724 | // Compute interesting factors from the set of interesting strides. | |||
2725 | for (SmallSetVector<const SCEV *, 4>::const_iterator | |||
2726 | I = Strides.begin(), E = Strides.end(); I != E; ++I) | |||
2727 | for (SmallSetVector<const SCEV *, 4>::const_iterator NewStrideIter = | |||
2728 | std::next(I); NewStrideIter != E; ++NewStrideIter) { | |||
2729 | const SCEV *OldStride = *I; | |||
2730 | const SCEV *NewStride = *NewStrideIter; | |||
2731 | ||||
2732 | if (SE.getTypeSizeInBits(OldStride->getType()) != | |||
2733 | SE.getTypeSizeInBits(NewStride->getType())) { | |||
2734 | if (SE.getTypeSizeInBits(OldStride->getType()) > | |||
2735 | SE.getTypeSizeInBits(NewStride->getType())) | |||
2736 | NewStride = SE.getSignExtendExpr(NewStride, OldStride->getType()); | |||
2737 | else | |||
2738 | OldStride = SE.getSignExtendExpr(OldStride, NewStride->getType()); | |||
2739 | } | |||
2740 | if (const SCEVConstant *Factor = | |||
2741 | dyn_cast_or_null<SCEVConstant>(getExactSDiv(NewStride, OldStride, | |||
2742 | SE, true))) { | |||
2743 | if (Factor->getAPInt().getMinSignedBits() <= 64 && !Factor->isZero()) | |||
2744 | Factors.insert(Factor->getAPInt().getSExtValue()); | |||
2745 | } else if (const SCEVConstant *Factor = | |||
2746 | dyn_cast_or_null<SCEVConstant>(getExactSDiv(OldStride, | |||
2747 | NewStride, | |||
2748 | SE, true))) { | |||
2749 | if (Factor->getAPInt().getMinSignedBits() <= 64 && !Factor->isZero()) | |||
2750 | Factors.insert(Factor->getAPInt().getSExtValue()); | |||
2751 | } | |||
2752 | } | |||
2753 | ||||
2754 | // If all uses use the same type, don't bother looking for truncation-based | |||
2755 | // reuse. | |||
2756 | if (Types.size() == 1) | |||
2757 | Types.clear(); | |||
2758 | ||||
2759 | LLVM_DEBUG(print_factors_and_types(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { print_factors_and_types(dbgs()); } } while (false); | |||
2760 | } | |||
2761 | ||||
2762 | /// Helper for CollectChains that finds an IV operand (computed by an AddRec in | |||
2763 | /// this loop) within [OI,OE) or returns OE. If IVUsers mapped Instructions to | |||
2764 | /// IVStrideUses, we could partially skip this. | |||
2765 | static User::op_iterator | |||
2766 | findIVOperand(User::op_iterator OI, User::op_iterator OE, | |||
2767 | Loop *L, ScalarEvolution &SE) { | |||
2768 | for(; OI != OE; ++OI) { | |||
2769 | if (Instruction *Oper = dyn_cast<Instruction>(*OI)) { | |||
2770 | if (!SE.isSCEVable(Oper->getType())) | |||
2771 | continue; | |||
2772 | ||||
2773 | if (const SCEVAddRecExpr *AR = | |||
2774 | dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Oper))) { | |||
2775 | if (AR->getLoop() == L) | |||
2776 | break; | |||
2777 | } | |||
2778 | } | |||
2779 | } | |||
2780 | return OI; | |||
2781 | } | |||
2782 | ||||
2783 | /// IVChain logic must consistently peek base TruncInst operands, so wrap it in | |||
2784 | /// a convenient helper. | |||
2785 | static Value *getWideOperand(Value *Oper) { | |||
2786 | if (TruncInst *Trunc = dyn_cast<TruncInst>(Oper)) | |||
2787 | return Trunc->getOperand(0); | |||
2788 | return Oper; | |||
2789 | } | |||
2790 | ||||
2791 | /// Return true if we allow an IV chain to include both types. | |||
2792 | static bool isCompatibleIVType(Value *LVal, Value *RVal) { | |||
2793 | Type *LType = LVal->getType(); | |||
2794 | Type *RType = RVal->getType(); | |||
2795 | return (LType == RType) || (LType->isPointerTy() && RType->isPointerTy() && | |||
2796 | // Different address spaces means (possibly) | |||
2797 | // different types of the pointer implementation, | |||
2798 | // e.g. i16 vs i32 so disallow that. | |||
2799 | (LType->getPointerAddressSpace() == | |||
2800 | RType->getPointerAddressSpace())); | |||
2801 | } | |||
2802 | ||||
2803 | /// Return an approximation of this SCEV expression's "base", or NULL for any | |||
2804 | /// constant. Returning the expression itself is conservative. Returning a | |||
2805 | /// deeper subexpression is more precise and valid as long as it isn't less | |||
2806 | /// complex than another subexpression. For expressions involving multiple | |||
2807 | /// unscaled values, we need to return the pointer-type SCEVUnknown. This avoids | |||
2808 | /// forming chains across objects, such as: PrevOper==a[i], IVOper==b[i], | |||
2809 | /// IVInc==b-a. | |||
2810 | /// | |||
2811 | /// Since SCEVUnknown is the rightmost type, and pointers are the rightmost | |||
2812 | /// SCEVUnknown, we simply return the rightmost SCEV operand. | |||
2813 | static const SCEV *getExprBase(const SCEV *S) { | |||
2814 | switch (S->getSCEVType()) { | |||
2815 | default: // uncluding scUnknown. | |||
2816 | return S; | |||
2817 | case scConstant: | |||
2818 | return nullptr; | |||
2819 | case scTruncate: | |||
2820 | return getExprBase(cast<SCEVTruncateExpr>(S)->getOperand()); | |||
2821 | case scZeroExtend: | |||
2822 | return getExprBase(cast<SCEVZeroExtendExpr>(S)->getOperand()); | |||
2823 | case scSignExtend: | |||
2824 | return getExprBase(cast<SCEVSignExtendExpr>(S)->getOperand()); | |||
2825 | case scAddExpr: { | |||
2826 | // Skip over scaled operands (scMulExpr) to follow add operands as long as | |||
2827 | // there's nothing more complex. | |||
2828 | // FIXME: not sure if we want to recognize negation. | |||
2829 | const SCEVAddExpr *Add = cast<SCEVAddExpr>(S); | |||
2830 | for (const SCEV *SubExpr : reverse(Add->operands())) { | |||
2831 | if (SubExpr->getSCEVType() == scAddExpr) | |||
2832 | return getExprBase(SubExpr); | |||
2833 | ||||
2834 | if (SubExpr->getSCEVType() != scMulExpr) | |||
2835 | return SubExpr; | |||
2836 | } | |||
2837 | return S; // all operands are scaled, be conservative. | |||
2838 | } | |||
2839 | case scAddRecExpr: | |||
2840 | return getExprBase(cast<SCEVAddRecExpr>(S)->getStart()); | |||
2841 | } | |||
2842 | llvm_unreachable("Unknown SCEV kind!")::llvm::llvm_unreachable_internal("Unknown SCEV kind!", "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp" , 2842); | |||
2843 | } | |||
2844 | ||||
2845 | /// Return true if the chain increment is profitable to expand into a loop | |||
2846 | /// invariant value, which may require its own register. A profitable chain | |||
2847 | /// increment will be an offset relative to the same base. We allow such offsets | |||
2848 | /// to potentially be used as chain increment as long as it's not obviously | |||
2849 | /// expensive to expand using real instructions. | |||
2850 | bool IVChain::isProfitableIncrement(const SCEV *OperExpr, | |||
2851 | const SCEV *IncExpr, | |||
2852 | ScalarEvolution &SE) { | |||
2853 | // Aggressively form chains when -stress-ivchain. | |||
2854 | if (StressIVChain) | |||
2855 | return true; | |||
2856 | ||||
2857 | // Do not replace a constant offset from IV head with a nonconstant IV | |||
2858 | // increment. | |||
2859 | if (!isa<SCEVConstant>(IncExpr)) { | |||
2860 | const SCEV *HeadExpr = SE.getSCEV(getWideOperand(Incs[0].IVOperand)); | |||
2861 | if (isa<SCEVConstant>(SE.getMinusSCEV(OperExpr, HeadExpr))) | |||
2862 | return false; | |||
2863 | } | |||
2864 | ||||
2865 | SmallPtrSet<const SCEV*, 8> Processed; | |||
2866 | return !isHighCostExpansion(IncExpr, Processed, SE); | |||
2867 | } | |||
2868 | ||||
2869 | /// Return true if the number of registers needed for the chain is estimated to | |||
2870 | /// be less than the number required for the individual IV users. First prohibit | |||
2871 | /// any IV users that keep the IV live across increments (the Users set should | |||
2872 | /// be empty). Next count the number and type of increments in the chain. | |||
2873 | /// | |||
2874 | /// Chaining IVs can lead to considerable code bloat if ISEL doesn't | |||
2875 | /// effectively use postinc addressing modes. Only consider it profitable it the | |||
2876 | /// increments can be computed in fewer registers when chained. | |||
2877 | /// | |||
2878 | /// TODO: Consider IVInc free if it's already used in another chains. | |||
2879 | static bool isProfitableChain(IVChain &Chain, | |||
2880 | SmallPtrSetImpl<Instruction *> &Users, | |||
2881 | ScalarEvolution &SE, | |||
2882 | const TargetTransformInfo &TTI) { | |||
2883 | if (StressIVChain) | |||
2884 | return true; | |||
2885 | ||||
2886 | if (!Chain.hasIncs()) | |||
2887 | return false; | |||
2888 | ||||
2889 | if (!Users.empty()) { | |||
2890 | LLVM_DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " users:\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Chain: " << *Chain. Incs[0].UserInst << " users:\n"; for (Instruction *Inst : Users) { dbgs() << " " << *Inst << "\n" ; }; } } while (false) | |||
2891 | for (Instruction *Instdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Chain: " << *Chain. Incs[0].UserInst << " users:\n"; for (Instruction *Inst : Users) { dbgs() << " " << *Inst << "\n" ; }; } } while (false) | |||
2892 | : Users) { dbgs() << " " << *Inst << "\n"; })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Chain: " << *Chain. Incs[0].UserInst << " users:\n"; for (Instruction *Inst : Users) { dbgs() << " " << *Inst << "\n" ; }; } } while (false); | |||
2893 | return false; | |||
2894 | } | |||
2895 | assert(!Chain.Incs.empty() && "empty IV chains are not allowed")(static_cast <bool> (!Chain.Incs.empty() && "empty IV chains are not allowed" ) ? void (0) : __assert_fail ("!Chain.Incs.empty() && \"empty IV chains are not allowed\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 2895, __extension__ __PRETTY_FUNCTION__)); | |||
2896 | ||||
2897 | // The chain itself may require a register, so intialize cost to 1. | |||
2898 | int cost = 1; | |||
2899 | ||||
2900 | // A complete chain likely eliminates the need for keeping the original IV in | |||
2901 | // a register. LSR does not currently know how to form a complete chain unless | |||
2902 | // the header phi already exists. | |||
2903 | if (isa<PHINode>(Chain.tailUserInst()) | |||
2904 | && SE.getSCEV(Chain.tailUserInst()) == Chain.Incs[0].IncExpr) { | |||
2905 | --cost; | |||
2906 | } | |||
2907 | const SCEV *LastIncExpr = nullptr; | |||
2908 | unsigned NumConstIncrements = 0; | |||
2909 | unsigned NumVarIncrements = 0; | |||
2910 | unsigned NumReusedIncrements = 0; | |||
2911 | ||||
2912 | if (TTI.isProfitableLSRChainElement(Chain.Incs[0].UserInst)) | |||
2913 | return true; | |||
2914 | ||||
2915 | for (const IVInc &Inc : Chain) { | |||
2916 | if (TTI.isProfitableLSRChainElement(Inc.UserInst)) | |||
2917 | return true; | |||
2918 | if (Inc.IncExpr->isZero()) | |||
2919 | continue; | |||
2920 | ||||
2921 | // Incrementing by zero or some constant is neutral. We assume constants can | |||
2922 | // be folded into an addressing mode or an add's immediate operand. | |||
2923 | if (isa<SCEVConstant>(Inc.IncExpr)) { | |||
2924 | ++NumConstIncrements; | |||
2925 | continue; | |||
2926 | } | |||
2927 | ||||
2928 | if (Inc.IncExpr == LastIncExpr) | |||
2929 | ++NumReusedIncrements; | |||
2930 | else | |||
2931 | ++NumVarIncrements; | |||
2932 | ||||
2933 | LastIncExpr = Inc.IncExpr; | |||
2934 | } | |||
2935 | // An IV chain with a single increment is handled by LSR's postinc | |||
2936 | // uses. However, a chain with multiple increments requires keeping the IV's | |||
2937 | // value live longer than it needs to be if chained. | |||
2938 | if (NumConstIncrements > 1) | |||
2939 | --cost; | |||
2940 | ||||
2941 | // Materializing increment expressions in the preheader that didn't exist in | |||
2942 | // the original code may cost a register. For example, sign-extended array | |||
2943 | // indices can produce ridiculous increments like this: | |||
2944 | // IV + ((sext i32 (2 * %s) to i64) + (-1 * (sext i32 %s to i64))) | |||
2945 | cost += NumVarIncrements; | |||
2946 | ||||
2947 | // Reusing variable increments likely saves a register to hold the multiple of | |||
2948 | // the stride. | |||
2949 | cost -= NumReusedIncrements; | |||
2950 | ||||
2951 | LLVM_DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " Cost: " << costdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Chain: " << *Chain. Incs[0].UserInst << " Cost: " << cost << "\n" ; } } while (false) | |||
2952 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Chain: " << *Chain. Incs[0].UserInst << " Cost: " << cost << "\n" ; } } while (false); | |||
2953 | ||||
2954 | return cost < 0; | |||
2955 | } | |||
2956 | ||||
2957 | /// Add this IV user to an existing chain or make it the head of a new chain. | |||
2958 | void LSRInstance::ChainInstruction(Instruction *UserInst, Instruction *IVOper, | |||
2959 | SmallVectorImpl<ChainUsers> &ChainUsersVec) { | |||
2960 | // When IVs are used as types of varying widths, they are generally converted | |||
2961 | // to a wider type with some uses remaining narrow under a (free) trunc. | |||
2962 | Value *const NextIV = getWideOperand(IVOper); | |||
2963 | const SCEV *const OperExpr = SE.getSCEV(NextIV); | |||
2964 | const SCEV *const OperExprBase = getExprBase(OperExpr); | |||
2965 | ||||
2966 | // Visit all existing chains. Check if its IVOper can be computed as a | |||
2967 | // profitable loop invariant increment from the last link in the Chain. | |||
2968 | unsigned ChainIdx = 0, NChains = IVChainVec.size(); | |||
2969 | const SCEV *LastIncExpr = nullptr; | |||
2970 | for (; ChainIdx < NChains; ++ChainIdx) { | |||
2971 | IVChain &Chain = IVChainVec[ChainIdx]; | |||
2972 | ||||
2973 | // Prune the solution space aggressively by checking that both IV operands | |||
2974 | // are expressions that operate on the same unscaled SCEVUnknown. This | |||
2975 | // "base" will be canceled by the subsequent getMinusSCEV call. Checking | |||
2976 | // first avoids creating extra SCEV expressions. | |||
2977 | if (!StressIVChain && Chain.ExprBase != OperExprBase) | |||
2978 | continue; | |||
2979 | ||||
2980 | Value *PrevIV = getWideOperand(Chain.Incs.back().IVOperand); | |||
2981 | if (!isCompatibleIVType(PrevIV, NextIV)) | |||
2982 | continue; | |||
2983 | ||||
2984 | // A phi node terminates a chain. | |||
2985 | if (isa<PHINode>(UserInst) && isa<PHINode>(Chain.tailUserInst())) | |||
2986 | continue; | |||
2987 | ||||
2988 | // The increment must be loop-invariant so it can be kept in a register. | |||
2989 | const SCEV *PrevExpr = SE.getSCEV(PrevIV); | |||
2990 | const SCEV *IncExpr = SE.getMinusSCEV(OperExpr, PrevExpr); | |||
2991 | if (isa<SCEVCouldNotCompute>(IncExpr) || !SE.isLoopInvariant(IncExpr, L)) | |||
2992 | continue; | |||
2993 | ||||
2994 | if (Chain.isProfitableIncrement(OperExpr, IncExpr, SE)) { | |||
2995 | LastIncExpr = IncExpr; | |||
2996 | break; | |||
2997 | } | |||
2998 | } | |||
2999 | // If we haven't found a chain, create a new one, unless we hit the max. Don't | |||
3000 | // bother for phi nodes, because they must be last in the chain. | |||
3001 | if (ChainIdx == NChains) { | |||
3002 | if (isa<PHINode>(UserInst)) | |||
3003 | return; | |||
3004 | if (NChains >= MaxChains && !StressIVChain) { | |||
3005 | LLVM_DEBUG(dbgs() << "IV Chain Limit\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "IV Chain Limit\n"; } } while (false); | |||
3006 | return; | |||
3007 | } | |||
3008 | LastIncExpr = OperExpr; | |||
3009 | // IVUsers may have skipped over sign/zero extensions. We don't currently | |||
3010 | // attempt to form chains involving extensions unless they can be hoisted | |||
3011 | // into this loop's AddRec. | |||
3012 | if (!isa<SCEVAddRecExpr>(LastIncExpr)) | |||
3013 | return; | |||
3014 | ++NChains; | |||
3015 | IVChainVec.push_back(IVChain(IVInc(UserInst, IVOper, LastIncExpr), | |||
3016 | OperExprBase)); | |||
3017 | ChainUsersVec.resize(NChains); | |||
3018 | LLVM_DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Head: (" << *UserInstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "IV Chain#" << ChainIdx << " Head: (" << *UserInst << ") IV=" << *LastIncExpr << "\n"; } } while (false) | |||
3019 | << ") IV=" << *LastIncExpr << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "IV Chain#" << ChainIdx << " Head: (" << *UserInst << ") IV=" << *LastIncExpr << "\n"; } } while (false); | |||
3020 | } else { | |||
3021 | LLVM_DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Inc: (" << *UserInstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "IV Chain#" << ChainIdx << " Inc: (" << *UserInst << ") IV+" << *LastIncExpr << "\n"; } } while (false) | |||
3022 | << ") IV+" << *LastIncExpr << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "IV Chain#" << ChainIdx << " Inc: (" << *UserInst << ") IV+" << *LastIncExpr << "\n"; } } while (false); | |||
3023 | // Add this IV user to the end of the chain. | |||
3024 | IVChainVec[ChainIdx].add(IVInc(UserInst, IVOper, LastIncExpr)); | |||
3025 | } | |||
3026 | IVChain &Chain = IVChainVec[ChainIdx]; | |||
3027 | ||||
3028 | SmallPtrSet<Instruction*,4> &NearUsers = ChainUsersVec[ChainIdx].NearUsers; | |||
3029 | // This chain's NearUsers become FarUsers. | |||
3030 | if (!LastIncExpr->isZero()) { | |||
3031 | ChainUsersVec[ChainIdx].FarUsers.insert(NearUsers.begin(), | |||
3032 | NearUsers.end()); | |||
3033 | NearUsers.clear(); | |||
3034 | } | |||
3035 | ||||
3036 | // All other uses of IVOperand become near uses of the chain. | |||
3037 | // We currently ignore intermediate values within SCEV expressions, assuming | |||
3038 | // they will eventually be used be the current chain, or can be computed | |||
3039 | // from one of the chain increments. To be more precise we could | |||
3040 | // transitively follow its user and only add leaf IV users to the set. | |||
3041 | for (User *U : IVOper->users()) { | |||
3042 | Instruction *OtherUse = dyn_cast<Instruction>(U); | |||
3043 | if (!OtherUse) | |||
3044 | continue; | |||
3045 | // Uses in the chain will no longer be uses if the chain is formed. | |||
3046 | // Include the head of the chain in this iteration (not Chain.begin()). | |||
3047 | IVChain::const_iterator IncIter = Chain.Incs.begin(); | |||
3048 | IVChain::const_iterator IncEnd = Chain.Incs.end(); | |||
3049 | for( ; IncIter != IncEnd; ++IncIter) { | |||
3050 | if (IncIter->UserInst == OtherUse) | |||
3051 | break; | |||
3052 | } | |||
3053 | if (IncIter != IncEnd) | |||
3054 | continue; | |||
3055 | ||||
3056 | if (SE.isSCEVable(OtherUse->getType()) | |||
3057 | && !isa<SCEVUnknown>(SE.getSCEV(OtherUse)) | |||
3058 | && IU.isIVUserOrOperand(OtherUse)) { | |||
3059 | continue; | |||
3060 | } | |||
3061 | NearUsers.insert(OtherUse); | |||
3062 | } | |||
3063 | ||||
3064 | // Since this user is part of the chain, it's no longer considered a use | |||
3065 | // of the chain. | |||
3066 | ChainUsersVec[ChainIdx].FarUsers.erase(UserInst); | |||
3067 | } | |||
3068 | ||||
3069 | /// Populate the vector of Chains. | |||
3070 | /// | |||
3071 | /// This decreases ILP at the architecture level. Targets with ample registers, | |||
3072 | /// multiple memory ports, and no register renaming probably don't want | |||
3073 | /// this. However, such targets should probably disable LSR altogether. | |||
3074 | /// | |||
3075 | /// The job of LSR is to make a reasonable choice of induction variables across | |||
3076 | /// the loop. Subsequent passes can easily "unchain" computation exposing more | |||
3077 | /// ILP *within the loop* if the target wants it. | |||
3078 | /// | |||
3079 | /// Finding the best IV chain is potentially a scheduling problem. Since LSR | |||
3080 | /// will not reorder memory operations, it will recognize this as a chain, but | |||
3081 | /// will generate redundant IV increments. Ideally this would be corrected later | |||
3082 | /// by a smart scheduler: | |||
3083 | /// = A[i] | |||
3084 | /// = A[i+x] | |||
3085 | /// A[i] = | |||
3086 | /// A[i+x] = | |||
3087 | /// | |||
3088 | /// TODO: Walk the entire domtree within this loop, not just the path to the | |||
3089 | /// loop latch. This will discover chains on side paths, but requires | |||
3090 | /// maintaining multiple copies of the Chains state. | |||
3091 | void LSRInstance::CollectChains() { | |||
3092 | LLVM_DEBUG(dbgs() << "Collecting IV Chains.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Collecting IV Chains.\n"; } } while (false); | |||
3093 | SmallVector<ChainUsers, 8> ChainUsersVec; | |||
3094 | ||||
3095 | SmallVector<BasicBlock *,8> LatchPath; | |||
3096 | BasicBlock *LoopHeader = L->getHeader(); | |||
3097 | for (DomTreeNode *Rung = DT.getNode(L->getLoopLatch()); | |||
3098 | Rung->getBlock() != LoopHeader; Rung = Rung->getIDom()) { | |||
3099 | LatchPath.push_back(Rung->getBlock()); | |||
3100 | } | |||
3101 | LatchPath.push_back(LoopHeader); | |||
3102 | ||||
3103 | // Walk the instruction stream from the loop header to the loop latch. | |||
3104 | for (BasicBlock *BB : reverse(LatchPath)) { | |||
3105 | for (Instruction &I : *BB) { | |||
3106 | // Skip instructions that weren't seen by IVUsers analysis. | |||
3107 | if (isa<PHINode>(I) || !IU.isIVUserOrOperand(&I)) | |||
3108 | continue; | |||
3109 | ||||
3110 | // Ignore users that are part of a SCEV expression. This way we only | |||
3111 | // consider leaf IV Users. This effectively rediscovers a portion of | |||
3112 | // IVUsers analysis but in program order this time. | |||
3113 | if (SE.isSCEVable(I.getType()) && !isa<SCEVUnknown>(SE.getSCEV(&I))) | |||
3114 | continue; | |||
3115 | ||||
3116 | // Remove this instruction from any NearUsers set it may be in. | |||
3117 | for (unsigned ChainIdx = 0, NChains = IVChainVec.size(); | |||
3118 | ChainIdx < NChains; ++ChainIdx) { | |||
3119 | ChainUsersVec[ChainIdx].NearUsers.erase(&I); | |||
3120 | } | |||
3121 | // Search for operands that can be chained. | |||
3122 | SmallPtrSet<Instruction*, 4> UniqueOperands; | |||
3123 | User::op_iterator IVOpEnd = I.op_end(); | |||
3124 | User::op_iterator IVOpIter = findIVOperand(I.op_begin(), IVOpEnd, L, SE); | |||
3125 | while (IVOpIter != IVOpEnd) { | |||
3126 | Instruction *IVOpInst = cast<Instruction>(*IVOpIter); | |||
3127 | if (UniqueOperands.insert(IVOpInst).second) | |||
3128 | ChainInstruction(&I, IVOpInst, ChainUsersVec); | |||
3129 | IVOpIter = findIVOperand(std::next(IVOpIter), IVOpEnd, L, SE); | |||
3130 | } | |||
3131 | } // Continue walking down the instructions. | |||
3132 | } // Continue walking down the domtree. | |||
3133 | // Visit phi backedges to determine if the chain can generate the IV postinc. | |||
3134 | for (PHINode &PN : L->getHeader()->phis()) { | |||
3135 | if (!SE.isSCEVable(PN.getType())) | |||
3136 | continue; | |||
3137 | ||||
3138 | Instruction *IncV = | |||
3139 | dyn_cast<Instruction>(PN.getIncomingValueForBlock(L->getLoopLatch())); | |||
3140 | if (IncV) | |||
3141 | ChainInstruction(&PN, IncV, ChainUsersVec); | |||
3142 | } | |||
3143 | // Remove any unprofitable chains. | |||
3144 | unsigned ChainIdx = 0; | |||
3145 | for (unsigned UsersIdx = 0, NChains = IVChainVec.size(); | |||
3146 | UsersIdx < NChains; ++UsersIdx) { | |||
3147 | if (!isProfitableChain(IVChainVec[UsersIdx], | |||
3148 | ChainUsersVec[UsersIdx].FarUsers, SE, TTI)) | |||
3149 | continue; | |||
3150 | // Preserve the chain at UsesIdx. | |||
3151 | if (ChainIdx != UsersIdx) | |||
3152 | IVChainVec[ChainIdx] = IVChainVec[UsersIdx]; | |||
3153 | FinalizeChain(IVChainVec[ChainIdx]); | |||
3154 | ++ChainIdx; | |||
3155 | } | |||
3156 | IVChainVec.resize(ChainIdx); | |||
3157 | } | |||
3158 | ||||
3159 | void LSRInstance::FinalizeChain(IVChain &Chain) { | |||
3160 | assert(!Chain.Incs.empty() && "empty IV chains are not allowed")(static_cast <bool> (!Chain.Incs.empty() && "empty IV chains are not allowed" ) ? void (0) : __assert_fail ("!Chain.Incs.empty() && \"empty IV chains are not allowed\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3160, __extension__ __PRETTY_FUNCTION__)); | |||
3161 | LLVM_DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[0].UserInst << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Final Chain: " << * Chain.Incs[0].UserInst << "\n"; } } while (false); | |||
3162 | ||||
3163 | for (const IVInc &Inc : Chain) { | |||
3164 | LLVM_DEBUG(dbgs() << " Inc: " << *Inc.UserInst << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Inc: " << * Inc.UserInst << "\n"; } } while (false); | |||
3165 | auto UseI = find(Inc.UserInst->operands(), Inc.IVOperand); | |||
3166 | assert(UseI != Inc.UserInst->op_end() && "cannot find IV operand")(static_cast <bool> (UseI != Inc.UserInst->op_end() && "cannot find IV operand") ? void (0) : __assert_fail ("UseI != Inc.UserInst->op_end() && \"cannot find IV operand\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3166, __extension__ __PRETTY_FUNCTION__)); | |||
3167 | IVIncSet.insert(UseI); | |||
3168 | } | |||
3169 | } | |||
3170 | ||||
3171 | /// Return true if the IVInc can be folded into an addressing mode. | |||
3172 | static bool canFoldIVIncExpr(const SCEV *IncExpr, Instruction *UserInst, | |||
3173 | Value *Operand, const TargetTransformInfo &TTI) { | |||
3174 | const SCEVConstant *IncConst = dyn_cast<SCEVConstant>(IncExpr); | |||
3175 | if (!IncConst || !isAddressUse(TTI, UserInst, Operand)) | |||
3176 | return false; | |||
3177 | ||||
3178 | if (IncConst->getAPInt().getMinSignedBits() > 64) | |||
3179 | return false; | |||
3180 | ||||
3181 | MemAccessTy AccessTy = getAccessType(TTI, UserInst, Operand); | |||
3182 | int64_t IncOffset = IncConst->getValue()->getSExtValue(); | |||
3183 | if (!isAlwaysFoldable(TTI, LSRUse::Address, AccessTy, /*BaseGV=*/nullptr, | |||
3184 | IncOffset, /*HasBaseReg=*/false)) | |||
3185 | return false; | |||
3186 | ||||
3187 | return true; | |||
3188 | } | |||
3189 | ||||
3190 | /// Generate an add or subtract for each IVInc in a chain to materialize the IV | |||
3191 | /// user's operand from the previous IV user's operand. | |||
3192 | void LSRInstance::GenerateIVChain(const IVChain &Chain, | |||
3193 | SmallVectorImpl<WeakTrackingVH> &DeadInsts) { | |||
3194 | // Find the new IVOperand for the head of the chain. It may have been replaced | |||
3195 | // by LSR. | |||
3196 | const IVInc &Head = Chain.Incs[0]; | |||
3197 | User::op_iterator IVOpEnd = Head.UserInst->op_end(); | |||
3198 | // findIVOperand returns IVOpEnd if it can no longer find a valid IV user. | |||
3199 | User::op_iterator IVOpIter = findIVOperand(Head.UserInst->op_begin(), | |||
3200 | IVOpEnd, L, SE); | |||
3201 | Value *IVSrc = nullptr; | |||
3202 | while (IVOpIter != IVOpEnd) { | |||
3203 | IVSrc = getWideOperand(*IVOpIter); | |||
3204 | ||||
3205 | // If this operand computes the expression that the chain needs, we may use | |||
3206 | // it. (Check this after setting IVSrc which is used below.) | |||
3207 | // | |||
3208 | // Note that if Head.IncExpr is wider than IVSrc, then this phi is too | |||
3209 | // narrow for the chain, so we can no longer use it. We do allow using a | |||
3210 | // wider phi, assuming the LSR checked for free truncation. In that case we | |||
3211 | // should already have a truncate on this operand such that | |||
3212 | // getSCEV(IVSrc) == IncExpr. | |||
3213 | if (SE.getSCEV(*IVOpIter) == Head.IncExpr | |||
3214 | || SE.getSCEV(IVSrc) == Head.IncExpr) { | |||
3215 | break; | |||
3216 | } | |||
3217 | IVOpIter = findIVOperand(std::next(IVOpIter), IVOpEnd, L, SE); | |||
3218 | } | |||
3219 | if (IVOpIter == IVOpEnd) { | |||
3220 | // Gracefully give up on this chain. | |||
3221 | LLVM_DEBUG(dbgs() << "Concealed chain head: " << *Head.UserInst << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Concealed chain head: " << *Head.UserInst << "\n"; } } while (false); | |||
3222 | return; | |||
3223 | } | |||
3224 | assert(IVSrc && "Failed to find IV chain source")(static_cast <bool> (IVSrc && "Failed to find IV chain source" ) ? void (0) : __assert_fail ("IVSrc && \"Failed to find IV chain source\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3224, __extension__ __PRETTY_FUNCTION__)); | |||
3225 | ||||
3226 | LLVM_DEBUG(dbgs() << "Generate chain at: " << *IVSrc << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Generate chain at: " << *IVSrc << "\n"; } } while (false); | |||
3227 | Type *IVTy = IVSrc->getType(); | |||
3228 | Type *IntTy = SE.getEffectiveSCEVType(IVTy); | |||
3229 | const SCEV *LeftOverExpr = nullptr; | |||
3230 | for (const IVInc &Inc : Chain) { | |||
3231 | Instruction *InsertPt = Inc.UserInst; | |||
3232 | if (isa<PHINode>(InsertPt)) | |||
3233 | InsertPt = L->getLoopLatch()->getTerminator(); | |||
3234 | ||||
3235 | // IVOper will replace the current IV User's operand. IVSrc is the IV | |||
3236 | // value currently held in a register. | |||
3237 | Value *IVOper = IVSrc; | |||
3238 | if (!Inc.IncExpr->isZero()) { | |||
3239 | // IncExpr was the result of subtraction of two narrow values, so must | |||
3240 | // be signed. | |||
3241 | const SCEV *IncExpr = SE.getNoopOrSignExtend(Inc.IncExpr, IntTy); | |||
3242 | LeftOverExpr = LeftOverExpr ? | |||
3243 | SE.getAddExpr(LeftOverExpr, IncExpr) : IncExpr; | |||
3244 | } | |||
3245 | if (LeftOverExpr && !LeftOverExpr->isZero()) { | |||
3246 | // Expand the IV increment. | |||
3247 | Rewriter.clearPostInc(); | |||
3248 | Value *IncV = Rewriter.expandCodeFor(LeftOverExpr, IntTy, InsertPt); | |||
3249 | const SCEV *IVOperExpr = SE.getAddExpr(SE.getUnknown(IVSrc), | |||
3250 | SE.getUnknown(IncV)); | |||
3251 | IVOper = Rewriter.expandCodeFor(IVOperExpr, IVTy, InsertPt); | |||
3252 | ||||
3253 | // If an IV increment can't be folded, use it as the next IV value. | |||
3254 | if (!canFoldIVIncExpr(LeftOverExpr, Inc.UserInst, Inc.IVOperand, TTI)) { | |||
3255 | assert(IVTy == IVOper->getType() && "inconsistent IV increment type")(static_cast <bool> (IVTy == IVOper->getType() && "inconsistent IV increment type") ? void (0) : __assert_fail ("IVTy == IVOper->getType() && \"inconsistent IV increment type\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3255, __extension__ __PRETTY_FUNCTION__)); | |||
3256 | IVSrc = IVOper; | |||
3257 | LeftOverExpr = nullptr; | |||
3258 | } | |||
3259 | } | |||
3260 | Type *OperTy = Inc.IVOperand->getType(); | |||
3261 | if (IVTy != OperTy) { | |||
3262 | assert(SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) &&(static_cast <bool> (SE.getTypeSizeInBits(IVTy) >= SE .getTypeSizeInBits(OperTy) && "cannot extend a chained IV" ) ? void (0) : __assert_fail ("SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) && \"cannot extend a chained IV\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3263, __extension__ __PRETTY_FUNCTION__)) | |||
3263 | "cannot extend a chained IV")(static_cast <bool> (SE.getTypeSizeInBits(IVTy) >= SE .getTypeSizeInBits(OperTy) && "cannot extend a chained IV" ) ? void (0) : __assert_fail ("SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) && \"cannot extend a chained IV\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3263, __extension__ __PRETTY_FUNCTION__)); | |||
3264 | IRBuilder<> Builder(InsertPt); | |||
3265 | IVOper = Builder.CreateTruncOrBitCast(IVOper, OperTy, "lsr.chain"); | |||
3266 | } | |||
3267 | Inc.UserInst->replaceUsesOfWith(Inc.IVOperand, IVOper); | |||
3268 | if (auto *OperandIsInstr = dyn_cast<Instruction>(Inc.IVOperand)) | |||
3269 | DeadInsts.emplace_back(OperandIsInstr); | |||
3270 | } | |||
3271 | // If LSR created a new, wider phi, we may also replace its postinc. We only | |||
3272 | // do this if we also found a wide value for the head of the chain. | |||
3273 | if (isa<PHINode>(Chain.tailUserInst())) { | |||
3274 | for (PHINode &Phi : L->getHeader()->phis()) { | |||
3275 | if (!isCompatibleIVType(&Phi, IVSrc)) | |||
3276 | continue; | |||
3277 | Instruction *PostIncV = dyn_cast<Instruction>( | |||
3278 | Phi.getIncomingValueForBlock(L->getLoopLatch())); | |||
3279 | if (!PostIncV || (SE.getSCEV(PostIncV) != SE.getSCEV(IVSrc))) | |||
3280 | continue; | |||
3281 | Value *IVOper = IVSrc; | |||
3282 | Type *PostIncTy = PostIncV->getType(); | |||
3283 | if (IVTy != PostIncTy) { | |||
3284 | assert(PostIncTy->isPointerTy() && "mixing int/ptr IV types")(static_cast <bool> (PostIncTy->isPointerTy() && "mixing int/ptr IV types") ? void (0) : __assert_fail ("PostIncTy->isPointerTy() && \"mixing int/ptr IV types\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3284, __extension__ __PRETTY_FUNCTION__)); | |||
3285 | IRBuilder<> Builder(L->getLoopLatch()->getTerminator()); | |||
3286 | Builder.SetCurrentDebugLocation(PostIncV->getDebugLoc()); | |||
3287 | IVOper = Builder.CreatePointerCast(IVSrc, PostIncTy, "lsr.chain"); | |||
3288 | } | |||
3289 | Phi.replaceUsesOfWith(PostIncV, IVOper); | |||
3290 | DeadInsts.emplace_back(PostIncV); | |||
3291 | } | |||
3292 | } | |||
3293 | } | |||
3294 | ||||
3295 | void LSRInstance::CollectFixupsAndInitialFormulae() { | |||
3296 | BranchInst *ExitBranch = nullptr; | |||
3297 | bool SaveCmp = TTI.canSaveCmp(L, &ExitBranch, &SE, &LI, &DT, &AC, &TLI); | |||
3298 | ||||
3299 | // For calculating baseline cost | |||
3300 | SmallPtrSet<const SCEV *, 16> Regs; | |||
3301 | DenseSet<const SCEV *> VisitedRegs; | |||
3302 | DenseSet<size_t> VisitedLSRUse; | |||
3303 | ||||
3304 | for (const IVStrideUse &U : IU) { | |||
3305 | Instruction *UserInst = U.getUser(); | |||
3306 | // Skip IV users that are part of profitable IV Chains. | |||
3307 | User::op_iterator UseI = | |||
3308 | find(UserInst->operands(), U.getOperandValToReplace()); | |||
3309 | assert(UseI != UserInst->op_end() && "cannot find IV operand")(static_cast <bool> (UseI != UserInst->op_end() && "cannot find IV operand") ? void (0) : __assert_fail ("UseI != UserInst->op_end() && \"cannot find IV operand\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3309, __extension__ __PRETTY_FUNCTION__)); | |||
3310 | if (IVIncSet.count(UseI)) { | |||
3311 | LLVM_DEBUG(dbgs() << "Use is in profitable chain: " << **UseI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Use is in profitable chain: " << **UseI << '\n'; } } while (false); | |||
3312 | continue; | |||
3313 | } | |||
3314 | ||||
3315 | LSRUse::KindType Kind = LSRUse::Basic; | |||
3316 | MemAccessTy AccessTy; | |||
3317 | if (isAddressUse(TTI, UserInst, U.getOperandValToReplace())) { | |||
3318 | Kind = LSRUse::Address; | |||
3319 | AccessTy = getAccessType(TTI, UserInst, U.getOperandValToReplace()); | |||
3320 | } | |||
3321 | ||||
3322 | const SCEV *S = IU.getExpr(U); | |||
3323 | PostIncLoopSet TmpPostIncLoops = U.getPostIncLoops(); | |||
3324 | ||||
3325 | // Equality (== and !=) ICmps are special. We can rewrite (i == N) as | |||
3326 | // (N - i == 0), and this allows (N - i) to be the expression that we work | |||
3327 | // with rather than just N or i, so we can consider the register | |||
3328 | // requirements for both N and i at the same time. Limiting this code to | |||
3329 | // equality icmps is not a problem because all interesting loops use | |||
3330 | // equality icmps, thanks to IndVarSimplify. | |||
3331 | if (ICmpInst *CI = dyn_cast<ICmpInst>(UserInst)) { | |||
3332 | // If CI can be saved in some target, like replaced inside hardware loop | |||
3333 | // in PowerPC, no need to generate initial formulae for it. | |||
3334 | if (SaveCmp && CI == dyn_cast<ICmpInst>(ExitBranch->getCondition())) | |||
3335 | continue; | |||
3336 | if (CI->isEquality()) { | |||
3337 | // Swap the operands if needed to put the OperandValToReplace on the | |||
3338 | // left, for consistency. | |||
3339 | Value *NV = CI->getOperand(1); | |||
3340 | if (NV == U.getOperandValToReplace()) { | |||
3341 | CI->setOperand(1, CI->getOperand(0)); | |||
3342 | CI->setOperand(0, NV); | |||
3343 | NV = CI->getOperand(1); | |||
3344 | Changed = true; | |||
3345 | } | |||
3346 | ||||
3347 | // x == y --> x - y == 0 | |||
3348 | const SCEV *N = SE.getSCEV(NV); | |||
3349 | if (SE.isLoopInvariant(N, L) && Rewriter.isSafeToExpand(N) && | |||
3350 | (!NV->getType()->isPointerTy() || | |||
3351 | SE.getPointerBase(N) == SE.getPointerBase(S))) { | |||
3352 | // S is normalized, so normalize N before folding it into S | |||
3353 | // to keep the result normalized. | |||
3354 | N = normalizeForPostIncUse(N, TmpPostIncLoops, SE); | |||
3355 | Kind = LSRUse::ICmpZero; | |||
3356 | S = SE.getMinusSCEV(N, S); | |||
3357 | } else if (L->isLoopInvariant(NV) && | |||
3358 | (!isa<Instruction>(NV) || | |||
3359 | DT.dominates(cast<Instruction>(NV), L->getHeader())) && | |||
3360 | !NV->getType()->isPointerTy()) { | |||
3361 | // If we can't generally expand the expression (e.g. it contains | |||
3362 | // a divide), but it is already at a loop invariant point before the | |||
3363 | // loop, wrap it in an unknown (to prevent the expander from trying | |||
3364 | // to re-expand in a potentially unsafe way.) The restriction to | |||
3365 | // integer types is required because the unknown hides the base, and | |||
3366 | // SCEV can't compute the difference of two unknown pointers. | |||
3367 | N = SE.getUnknown(NV); | |||
3368 | N = normalizeForPostIncUse(N, TmpPostIncLoops, SE); | |||
3369 | Kind = LSRUse::ICmpZero; | |||
3370 | S = SE.getMinusSCEV(N, S); | |||
3371 | assert(!isa<SCEVCouldNotCompute>(S))(static_cast <bool> (!isa<SCEVCouldNotCompute>(S) ) ? void (0) : __assert_fail ("!isa<SCEVCouldNotCompute>(S)" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3371, __extension__ __PRETTY_FUNCTION__)); | |||
3372 | } | |||
3373 | ||||
3374 | // -1 and the negations of all interesting strides (except the negation | |||
3375 | // of -1) are now also interesting. | |||
3376 | for (size_t i = 0, e = Factors.size(); i != e; ++i) | |||
3377 | if (Factors[i] != -1) | |||
3378 | Factors.insert(-(uint64_t)Factors[i]); | |||
3379 | Factors.insert(-1); | |||
3380 | } | |||
3381 | } | |||
3382 | ||||
3383 | // Get or create an LSRUse. | |||
3384 | std::pair<size_t, int64_t> P = getUse(S, Kind, AccessTy); | |||
3385 | size_t LUIdx = P.first; | |||
3386 | int64_t Offset = P.second; | |||
3387 | LSRUse &LU = Uses[LUIdx]; | |||
3388 | ||||
3389 | // Record the fixup. | |||
3390 | LSRFixup &LF = LU.getNewFixup(); | |||
3391 | LF.UserInst = UserInst; | |||
3392 | LF.OperandValToReplace = U.getOperandValToReplace(); | |||
3393 | LF.PostIncLoops = TmpPostIncLoops; | |||
3394 | LF.Offset = Offset; | |||
3395 | LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); | |||
3396 | ||||
3397 | // Create SCEV as Formula for calculating baseline cost | |||
3398 | if (!VisitedLSRUse.count(LUIdx) && !LF.isUseFullyOutsideLoop(L)) { | |||
3399 | Formula F; | |||
3400 | F.initialMatch(S, L, SE); | |||
3401 | BaselineCost.RateFormula(F, Regs, VisitedRegs, LU); | |||
3402 | VisitedLSRUse.insert(LUIdx); | |||
3403 | } | |||
3404 | ||||
3405 | if (!LU.WidestFixupType || | |||
3406 | SE.getTypeSizeInBits(LU.WidestFixupType) < | |||
3407 | SE.getTypeSizeInBits(LF.OperandValToReplace->getType())) | |||
3408 | LU.WidestFixupType = LF.OperandValToReplace->getType(); | |||
3409 | ||||
3410 | // If this is the first use of this LSRUse, give it a formula. | |||
3411 | if (LU.Formulae.empty()) { | |||
3412 | InsertInitialFormula(S, LU, LUIdx); | |||
3413 | CountRegisters(LU.Formulae.back(), LUIdx); | |||
3414 | } | |||
3415 | } | |||
3416 | ||||
3417 | LLVM_DEBUG(print_fixups(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { print_fixups(dbgs()); } } while (false); | |||
3418 | } | |||
3419 | ||||
3420 | /// Insert a formula for the given expression into the given use, separating out | |||
3421 | /// loop-variant portions from loop-invariant and loop-computable portions. | |||
3422 | void LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, | |||
3423 | size_t LUIdx) { | |||
3424 | // Mark uses whose expressions cannot be expanded. | |||
3425 | if (!Rewriter.isSafeToExpand(S)) | |||
3426 | LU.RigidFormula = true; | |||
3427 | ||||
3428 | Formula F; | |||
3429 | F.initialMatch(S, L, SE); | |||
3430 | bool Inserted = InsertFormula(LU, LUIdx, F); | |||
3431 | assert(Inserted && "Initial formula already exists!")(static_cast <bool> (Inserted && "Initial formula already exists!" ) ? void (0) : __assert_fail ("Inserted && \"Initial formula already exists!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3431, __extension__ __PRETTY_FUNCTION__)); (void)Inserted; | |||
3432 | } | |||
3433 | ||||
3434 | /// Insert a simple single-register formula for the given expression into the | |||
3435 | /// given use. | |||
3436 | void | |||
3437 | LSRInstance::InsertSupplementalFormula(const SCEV *S, | |||
3438 | LSRUse &LU, size_t LUIdx) { | |||
3439 | Formula F; | |||
3440 | F.BaseRegs.push_back(S); | |||
3441 | F.HasBaseReg = true; | |||
3442 | bool Inserted = InsertFormula(LU, LUIdx, F); | |||
3443 | assert(Inserted && "Supplemental formula already exists!")(static_cast <bool> (Inserted && "Supplemental formula already exists!" ) ? void (0) : __assert_fail ("Inserted && \"Supplemental formula already exists!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3443, __extension__ __PRETTY_FUNCTION__)); (void)Inserted; | |||
3444 | } | |||
3445 | ||||
3446 | /// Note which registers are used by the given formula, updating RegUses. | |||
3447 | void LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) { | |||
3448 | if (F.ScaledReg) | |||
3449 | RegUses.countRegister(F.ScaledReg, LUIdx); | |||
3450 | for (const SCEV *BaseReg : F.BaseRegs) | |||
3451 | RegUses.countRegister(BaseReg, LUIdx); | |||
3452 | } | |||
3453 | ||||
3454 | /// If the given formula has not yet been inserted, add it to the list, and | |||
3455 | /// return true. Return false otherwise. | |||
3456 | bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) { | |||
3457 | // Do not insert formula that we will not be able to expand. | |||
3458 | assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&(static_cast <bool> (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset , LU.Kind, LU.AccessTy, F) && "Formula is illegal") ? void (0) : __assert_fail ("isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) && \"Formula is illegal\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3459, __extension__ __PRETTY_FUNCTION__)) | |||
3459 | "Formula is illegal")(static_cast <bool> (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset , LU.Kind, LU.AccessTy, F) && "Formula is illegal") ? void (0) : __assert_fail ("isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) && \"Formula is illegal\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3459, __extension__ __PRETTY_FUNCTION__)); | |||
3460 | ||||
3461 | if (!LU.InsertFormula(F, *L)) | |||
3462 | return false; | |||
3463 | ||||
3464 | CountRegisters(F, LUIdx); | |||
3465 | return true; | |||
3466 | } | |||
3467 | ||||
3468 | /// Check for other uses of loop-invariant values which we're tracking. These | |||
3469 | /// other uses will pin these values in registers, making them less profitable | |||
3470 | /// for elimination. | |||
3471 | /// TODO: This currently misses non-constant addrec step registers. | |||
3472 | /// TODO: Should this give more weight to users inside the loop? | |||
3473 | void | |||
3474 | LSRInstance::CollectLoopInvariantFixupsAndFormulae() { | |||
3475 | SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end()); | |||
3476 | SmallPtrSet<const SCEV *, 32> Visited; | |||
3477 | ||||
3478 | while (!Worklist.empty()) { | |||
3479 | const SCEV *S = Worklist.pop_back_val(); | |||
3480 | ||||
3481 | // Don't process the same SCEV twice | |||
3482 | if (!Visited.insert(S).second) | |||
3483 | continue; | |||
3484 | ||||
3485 | if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(S)) | |||
3486 | append_range(Worklist, N->operands()); | |||
3487 | else if (const SCEVIntegralCastExpr *C = dyn_cast<SCEVIntegralCastExpr>(S)) | |||
3488 | Worklist.push_back(C->getOperand()); | |||
3489 | else if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) { | |||
3490 | Worklist.push_back(D->getLHS()); | |||
3491 | Worklist.push_back(D->getRHS()); | |||
3492 | } else if (const SCEVUnknown *US = dyn_cast<SCEVUnknown>(S)) { | |||
3493 | const Value *V = US->getValue(); | |||
3494 | if (const Instruction *Inst = dyn_cast<Instruction>(V)) { | |||
3495 | // Look for instructions defined outside the loop. | |||
3496 | if (L->contains(Inst)) continue; | |||
3497 | } else if (isa<UndefValue>(V)) | |||
3498 | // Undef doesn't have a live range, so it doesn't matter. | |||
3499 | continue; | |||
3500 | for (const Use &U : V->uses()) { | |||
3501 | const Instruction *UserInst = dyn_cast<Instruction>(U.getUser()); | |||
3502 | // Ignore non-instructions. | |||
3503 | if (!UserInst) | |||
3504 | continue; | |||
3505 | // Don't bother if the instruction is an EHPad. | |||
3506 | if (UserInst->isEHPad()) | |||
3507 | continue; | |||
3508 | // Ignore instructions in other functions (as can happen with | |||
3509 | // Constants). | |||
3510 | if (UserInst->getParent()->getParent() != L->getHeader()->getParent()) | |||
3511 | continue; | |||
3512 | // Ignore instructions not dominated by the loop. | |||
3513 | const BasicBlock *UseBB = !isa<PHINode>(UserInst) ? | |||
3514 | UserInst->getParent() : | |||
3515 | cast<PHINode>(UserInst)->getIncomingBlock( | |||
3516 | PHINode::getIncomingValueNumForOperand(U.getOperandNo())); | |||
3517 | if (!DT.dominates(L->getHeader(), UseBB)) | |||
3518 | continue; | |||
3519 | // Don't bother if the instruction is in a BB which ends in an EHPad. | |||
3520 | if (UseBB->getTerminator()->isEHPad()) | |||
3521 | continue; | |||
3522 | ||||
3523 | // Ignore cases in which the currently-examined value could come from | |||
3524 | // a basic block terminated with an EHPad. This checks all incoming | |||
3525 | // blocks of the phi node since it is possible that the same incoming | |||
3526 | // value comes from multiple basic blocks, only some of which may end | |||
3527 | // in an EHPad. If any of them do, a subsequent rewrite attempt by this | |||
3528 | // pass would try to insert instructions into an EHPad, hitting an | |||
3529 | // assertion. | |||
3530 | if (isa<PHINode>(UserInst)) { | |||
3531 | const auto *PhiNode = cast<PHINode>(UserInst); | |||
3532 | bool HasIncompatibleEHPTerminatedBlock = false; | |||
3533 | llvm::Value *ExpectedValue = U; | |||
3534 | for (unsigned int I = 0; I < PhiNode->getNumIncomingValues(); I++) { | |||
3535 | if (PhiNode->getIncomingValue(I) == ExpectedValue) { | |||
3536 | if (PhiNode->getIncomingBlock(I)->getTerminator()->isEHPad()) { | |||
3537 | HasIncompatibleEHPTerminatedBlock = true; | |||
3538 | break; | |||
3539 | } | |||
3540 | } | |||
3541 | } | |||
3542 | if (HasIncompatibleEHPTerminatedBlock) { | |||
3543 | continue; | |||
3544 | } | |||
3545 | } | |||
3546 | ||||
3547 | // Don't bother rewriting PHIs in catchswitch blocks. | |||
3548 | if (isa<CatchSwitchInst>(UserInst->getParent()->getTerminator())) | |||
3549 | continue; | |||
3550 | // Ignore uses which are part of other SCEV expressions, to avoid | |||
3551 | // analyzing them multiple times. | |||
3552 | if (SE.isSCEVable(UserInst->getType())) { | |||
3553 | const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst)); | |||
3554 | // If the user is a no-op, look through to its uses. | |||
3555 | if (!isa<SCEVUnknown>(UserS)) | |||
3556 | continue; | |||
3557 | if (UserS == US) { | |||
3558 | Worklist.push_back( | |||
3559 | SE.getUnknown(const_cast<Instruction *>(UserInst))); | |||
3560 | continue; | |||
3561 | } | |||
3562 | } | |||
3563 | // Ignore icmp instructions which are already being analyzed. | |||
3564 | if (const ICmpInst *ICI = dyn_cast<ICmpInst>(UserInst)) { | |||
3565 | unsigned OtherIdx = !U.getOperandNo(); | |||
3566 | Value *OtherOp = const_cast<Value *>(ICI->getOperand(OtherIdx)); | |||
3567 | if (SE.hasComputableLoopEvolution(SE.getSCEV(OtherOp), L)) | |||
3568 | continue; | |||
3569 | } | |||
3570 | ||||
3571 | std::pair<size_t, int64_t> P = getUse( | |||
3572 | S, LSRUse::Basic, MemAccessTy()); | |||
3573 | size_t LUIdx = P.first; | |||
3574 | int64_t Offset = P.second; | |||
3575 | LSRUse &LU = Uses[LUIdx]; | |||
3576 | LSRFixup &LF = LU.getNewFixup(); | |||
3577 | LF.UserInst = const_cast<Instruction *>(UserInst); | |||
3578 | LF.OperandValToReplace = U; | |||
3579 | LF.Offset = Offset; | |||
3580 | LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L); | |||
3581 | if (!LU.WidestFixupType || | |||
3582 | SE.getTypeSizeInBits(LU.WidestFixupType) < | |||
3583 | SE.getTypeSizeInBits(LF.OperandValToReplace->getType())) | |||
3584 | LU.WidestFixupType = LF.OperandValToReplace->getType(); | |||
3585 | InsertSupplementalFormula(US, LU, LUIdx); | |||
3586 | CountRegisters(LU.Formulae.back(), Uses.size() - 1); | |||
3587 | break; | |||
3588 | } | |||
3589 | } | |||
3590 | } | |||
3591 | } | |||
3592 | ||||
3593 | /// Split S into subexpressions which can be pulled out into separate | |||
3594 | /// registers. If C is non-null, multiply each subexpression by C. | |||
3595 | /// | |||
3596 | /// Return remainder expression after factoring the subexpressions captured by | |||
3597 | /// Ops. If Ops is complete, return NULL. | |||
3598 | static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C, | |||
3599 | SmallVectorImpl<const SCEV *> &Ops, | |||
3600 | const Loop *L, | |||
3601 | ScalarEvolution &SE, | |||
3602 | unsigned Depth = 0) { | |||
3603 | // Arbitrarily cap recursion to protect compile time. | |||
3604 | if (Depth >= 3) | |||
3605 | return S; | |||
3606 | ||||
3607 | if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) { | |||
3608 | // Break out add operands. | |||
3609 | for (const SCEV *S : Add->operands()) { | |||
3610 | const SCEV *Remainder = CollectSubexprs(S, C, Ops, L, SE, Depth+1); | |||
3611 | if (Remainder) | |||
3612 | Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder); | |||
3613 | } | |||
3614 | return nullptr; | |||
3615 | } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) { | |||
3616 | // Split a non-zero base out of an addrec. | |||
3617 | if (AR->getStart()->isZero() || !AR->isAffine()) | |||
3618 | return S; | |||
3619 | ||||
3620 | const SCEV *Remainder = CollectSubexprs(AR->getStart(), | |||
3621 | C, Ops, L, SE, Depth+1); | |||
3622 | // Split the non-zero AddRec unless it is part of a nested recurrence that | |||
3623 | // does not pertain to this loop. | |||
3624 | if (Remainder && (AR->getLoop() == L || !isa<SCEVAddRecExpr>(Remainder))) { | |||
3625 | Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder); | |||
3626 | Remainder = nullptr; | |||
3627 | } | |||
3628 | if (Remainder != AR->getStart()) { | |||
3629 | if (!Remainder) | |||
3630 | Remainder = SE.getConstant(AR->getType(), 0); | |||
3631 | return SE.getAddRecExpr(Remainder, | |||
3632 | AR->getStepRecurrence(SE), | |||
3633 | AR->getLoop(), | |||
3634 | //FIXME: AR->getNoWrapFlags(SCEV::FlagNW) | |||
3635 | SCEV::FlagAnyWrap); | |||
3636 | } | |||
3637 | } else if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) { | |||
3638 | // Break (C * (a + b + c)) into C*a + C*b + C*c. | |||
3639 | if (Mul->getNumOperands() != 2) | |||
3640 | return S; | |||
3641 | if (const SCEVConstant *Op0 = | |||
3642 | dyn_cast<SCEVConstant>(Mul->getOperand(0))) { | |||
3643 | C = C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0; | |||
3644 | const SCEV *Remainder = | |||
3645 | CollectSubexprs(Mul->getOperand(1), C, Ops, L, SE, Depth+1); | |||
3646 | if (Remainder) | |||
3647 | Ops.push_back(SE.getMulExpr(C, Remainder)); | |||
3648 | return nullptr; | |||
3649 | } | |||
3650 | } | |||
3651 | return S; | |||
3652 | } | |||
3653 | ||||
3654 | /// Return true if the SCEV represents a value that may end up as a | |||
3655 | /// post-increment operation. | |||
3656 | static bool mayUsePostIncMode(const TargetTransformInfo &TTI, | |||
3657 | LSRUse &LU, const SCEV *S, const Loop *L, | |||
3658 | ScalarEvolution &SE) { | |||
3659 | if (LU.Kind != LSRUse::Address || | |||
3660 | !LU.AccessTy.getType()->isIntOrIntVectorTy()) | |||
3661 | return false; | |||
3662 | const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S); | |||
3663 | if (!AR) | |||
3664 | return false; | |||
3665 | const SCEV *LoopStep = AR->getStepRecurrence(SE); | |||
3666 | if (!isa<SCEVConstant>(LoopStep)) | |||
3667 | return false; | |||
3668 | // Check if a post-indexed load/store can be used. | |||
3669 | if (TTI.isIndexedLoadLegal(TTI.MIM_PostInc, AR->getType()) || | |||
3670 | TTI.isIndexedStoreLegal(TTI.MIM_PostInc, AR->getType())) { | |||
3671 | const SCEV *LoopStart = AR->getStart(); | |||
3672 | if (!isa<SCEVConstant>(LoopStart) && SE.isLoopInvariant(LoopStart, L)) | |||
3673 | return true; | |||
3674 | } | |||
3675 | return false; | |||
3676 | } | |||
3677 | ||||
3678 | /// Helper function for LSRInstance::GenerateReassociations. | |||
3679 | void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx, | |||
3680 | const Formula &Base, | |||
3681 | unsigned Depth, size_t Idx, | |||
3682 | bool IsScaledReg) { | |||
3683 | const SCEV *BaseReg = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx]; | |||
3684 | // Don't generate reassociations for the base register of a value that | |||
3685 | // may generate a post-increment operator. The reason is that the | |||
3686 | // reassociations cause extra base+register formula to be created, | |||
3687 | // and possibly chosen, but the post-increment is more efficient. | |||
3688 | if (AMK == TTI::AMK_PostIndexed && mayUsePostIncMode(TTI, LU, BaseReg, L, SE)) | |||
3689 | return; | |||
3690 | SmallVector<const SCEV *, 8> AddOps; | |||
3691 | const SCEV *Remainder = CollectSubexprs(BaseReg, nullptr, AddOps, L, SE); | |||
3692 | if (Remainder) | |||
3693 | AddOps.push_back(Remainder); | |||
3694 | ||||
3695 | if (AddOps.size() == 1) | |||
3696 | return; | |||
3697 | ||||
3698 | for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(), | |||
3699 | JE = AddOps.end(); | |||
3700 | J != JE; ++J) { | |||
3701 | // Loop-variant "unknown" values are uninteresting; we won't be able to | |||
3702 | // do anything meaningful with them. | |||
3703 | if (isa<SCEVUnknown>(*J) && !SE.isLoopInvariant(*J, L)) | |||
3704 | continue; | |||
3705 | ||||
3706 | // Don't pull a constant into a register if the constant could be folded | |||
3707 | // into an immediate field. | |||
3708 | if (isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind, | |||
3709 | LU.AccessTy, *J, Base.getNumRegs() > 1)) | |||
3710 | continue; | |||
3711 | ||||
3712 | // Collect all operands except *J. | |||
3713 | SmallVector<const SCEV *, 8> InnerAddOps( | |||
3714 | ((const SmallVector<const SCEV *, 8> &)AddOps).begin(), J); | |||
3715 | InnerAddOps.append(std::next(J), | |||
3716 | ((const SmallVector<const SCEV *, 8> &)AddOps).end()); | |||
3717 | ||||
3718 | // Don't leave just a constant behind in a register if the constant could | |||
3719 | // be folded into an immediate field. | |||
3720 | if (InnerAddOps.size() == 1 && | |||
3721 | isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind, | |||
3722 | LU.AccessTy, InnerAddOps[0], Base.getNumRegs() > 1)) | |||
3723 | continue; | |||
3724 | ||||
3725 | const SCEV *InnerSum = SE.getAddExpr(InnerAddOps); | |||
3726 | if (InnerSum->isZero()) | |||
3727 | continue; | |||
3728 | Formula F = Base; | |||
3729 | ||||
3730 | // Add the remaining pieces of the add back into the new formula. | |||
3731 | const SCEVConstant *InnerSumSC = dyn_cast<SCEVConstant>(InnerSum); | |||
3732 | if (InnerSumSC && SE.getTypeSizeInBits(InnerSumSC->getType()) <= 64 && | |||
3733 | TTI.isLegalAddImmediate((uint64_t)F.UnfoldedOffset + | |||
3734 | InnerSumSC->getValue()->getZExtValue())) { | |||
3735 | F.UnfoldedOffset = | |||
3736 | (uint64_t)F.UnfoldedOffset + InnerSumSC->getValue()->getZExtValue(); | |||
3737 | if (IsScaledReg) | |||
3738 | F.ScaledReg = nullptr; | |||
3739 | else | |||
3740 | F.BaseRegs.erase(F.BaseRegs.begin() + Idx); | |||
3741 | } else if (IsScaledReg) | |||
3742 | F.ScaledReg = InnerSum; | |||
3743 | else | |||
3744 | F.BaseRegs[Idx] = InnerSum; | |||
3745 | ||||
3746 | // Add J as its own register, or an unfolded immediate. | |||
3747 | const SCEVConstant *SC = dyn_cast<SCEVConstant>(*J); | |||
3748 | if (SC && SE.getTypeSizeInBits(SC->getType()) <= 64 && | |||
3749 | TTI.isLegalAddImmediate((uint64_t)F.UnfoldedOffset + | |||
3750 | SC->getValue()->getZExtValue())) | |||
3751 | F.UnfoldedOffset = | |||
3752 | (uint64_t)F.UnfoldedOffset + SC->getValue()->getZExtValue(); | |||
3753 | else | |||
3754 | F.BaseRegs.push_back(*J); | |||
3755 | // We may have changed the number of register in base regs, adjust the | |||
3756 | // formula accordingly. | |||
3757 | F.canonicalize(*L); | |||
3758 | ||||
3759 | if (InsertFormula(LU, LUIdx, F)) | |||
3760 | // If that formula hadn't been seen before, recurse to find more like | |||
3761 | // it. | |||
3762 | // Add check on Log16(AddOps.size()) - same as Log2_32(AddOps.size()) >> 2) | |||
3763 | // Because just Depth is not enough to bound compile time. | |||
3764 | // This means that every time AddOps.size() is greater 16^x we will add | |||
3765 | // x to Depth. | |||
3766 | GenerateReassociations(LU, LUIdx, LU.Formulae.back(), | |||
3767 | Depth + 1 + (Log2_32(AddOps.size()) >> 2)); | |||
3768 | } | |||
3769 | } | |||
3770 | ||||
3771 | /// Split out subexpressions from adds and the bases of addrecs. | |||
3772 | void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx, | |||
3773 | Formula Base, unsigned Depth) { | |||
3774 | assert(Base.isCanonical(*L) && "Input must be in the canonical form")(static_cast <bool> (Base.isCanonical(*L) && "Input must be in the canonical form" ) ? void (0) : __assert_fail ("Base.isCanonical(*L) && \"Input must be in the canonical form\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3774, __extension__ __PRETTY_FUNCTION__)); | |||
3775 | // Arbitrarily cap recursion to protect compile time. | |||
3776 | if (Depth >= 3) | |||
3777 | return; | |||
3778 | ||||
3779 | for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) | |||
3780 | GenerateReassociationsImpl(LU, LUIdx, Base, Depth, i); | |||
3781 | ||||
3782 | if (Base.Scale == 1) | |||
3783 | GenerateReassociationsImpl(LU, LUIdx, Base, Depth, | |||
3784 | /* Idx */ -1, /* IsScaledReg */ true); | |||
3785 | } | |||
3786 | ||||
3787 | /// Generate a formula consisting of all of the loop-dominating registers added | |||
3788 | /// into a single register. | |||
3789 | void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx, | |||
3790 | Formula Base) { | |||
3791 | // This method is only interesting on a plurality of registers. | |||
3792 | if (Base.BaseRegs.size() + (Base.Scale == 1) + | |||
3793 | (Base.UnfoldedOffset != 0) <= 1) | |||
3794 | return; | |||
3795 | ||||
3796 | // Flatten the representation, i.e., reg1 + 1*reg2 => reg1 + reg2, before | |||
3797 | // processing the formula. | |||
3798 | Base.unscale(); | |||
3799 | SmallVector<const SCEV *, 4> Ops; | |||
3800 | Formula NewBase = Base; | |||
3801 | NewBase.BaseRegs.clear(); | |||
3802 | Type *CombinedIntegerType = nullptr; | |||
3803 | for (const SCEV *BaseReg : Base.BaseRegs) { | |||
3804 | if (SE.properlyDominates(BaseReg, L->getHeader()) && | |||
3805 | !SE.hasComputableLoopEvolution(BaseReg, L)) { | |||
3806 | if (!CombinedIntegerType) | |||
3807 | CombinedIntegerType = SE.getEffectiveSCEVType(BaseReg->getType()); | |||
3808 | Ops.push_back(BaseReg); | |||
3809 | } | |||
3810 | else | |||
3811 | NewBase.BaseRegs.push_back(BaseReg); | |||
3812 | } | |||
3813 | ||||
3814 | // If no register is relevant, we're done. | |||
3815 | if (Ops.size() == 0) | |||
3816 | return; | |||
3817 | ||||
3818 | // Utility function for generating the required variants of the combined | |||
3819 | // registers. | |||
3820 | auto GenerateFormula = [&](const SCEV *Sum) { | |||
3821 | Formula F = NewBase; | |||
3822 | ||||
3823 | // TODO: If Sum is zero, it probably means ScalarEvolution missed an | |||
3824 | // opportunity to fold something. For now, just ignore such cases | |||
3825 | // rather than proceed with zero in a register. | |||
3826 | if (Sum->isZero()) | |||
3827 | return; | |||
3828 | ||||
3829 | F.BaseRegs.push_back(Sum); | |||
3830 | F.canonicalize(*L); | |||
3831 | (void)InsertFormula(LU, LUIdx, F); | |||
3832 | }; | |||
3833 | ||||
3834 | // If we collected at least two registers, generate a formula combining them. | |||
3835 | if (Ops.size() > 1) { | |||
3836 | SmallVector<const SCEV *, 4> OpsCopy(Ops); // Don't let SE modify Ops. | |||
3837 | GenerateFormula(SE.getAddExpr(OpsCopy)); | |||
3838 | } | |||
3839 | ||||
3840 | // If we have an unfolded offset, generate a formula combining it with the | |||
3841 | // registers collected. | |||
3842 | if (NewBase.UnfoldedOffset) { | |||
3843 | assert(CombinedIntegerType && "Missing a type for the unfolded offset")(static_cast <bool> (CombinedIntegerType && "Missing a type for the unfolded offset" ) ? void (0) : __assert_fail ("CombinedIntegerType && \"Missing a type for the unfolded offset\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3843, __extension__ __PRETTY_FUNCTION__)); | |||
3844 | Ops.push_back(SE.getConstant(CombinedIntegerType, NewBase.UnfoldedOffset, | |||
3845 | true)); | |||
3846 | NewBase.UnfoldedOffset = 0; | |||
3847 | GenerateFormula(SE.getAddExpr(Ops)); | |||
3848 | } | |||
3849 | } | |||
3850 | ||||
3851 | /// Helper function for LSRInstance::GenerateSymbolicOffsets. | |||
3852 | void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx, | |||
3853 | const Formula &Base, size_t Idx, | |||
3854 | bool IsScaledReg) { | |||
3855 | const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx]; | |||
3856 | GlobalValue *GV = ExtractSymbol(G, SE); | |||
3857 | if (G->isZero() || !GV) | |||
3858 | return; | |||
3859 | Formula F = Base; | |||
3860 | F.BaseGV = GV; | |||
3861 | if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) | |||
3862 | return; | |||
3863 | if (IsScaledReg) | |||
3864 | F.ScaledReg = G; | |||
3865 | else | |||
3866 | F.BaseRegs[Idx] = G; | |||
3867 | (void)InsertFormula(LU, LUIdx, F); | |||
3868 | } | |||
3869 | ||||
3870 | /// Generate reuse formulae using symbolic offsets. | |||
3871 | void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, | |||
3872 | Formula Base) { | |||
3873 | // We can't add a symbolic offset if the address already contains one. | |||
3874 | if (Base.BaseGV) return; | |||
3875 | ||||
3876 | for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) | |||
3877 | GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, i); | |||
3878 | if (Base.Scale == 1) | |||
3879 | GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1, | |||
3880 | /* IsScaledReg */ true); | |||
3881 | } | |||
3882 | ||||
3883 | /// Helper function for LSRInstance::GenerateConstantOffsets. | |||
3884 | void LSRInstance::GenerateConstantOffsetsImpl( | |||
3885 | LSRUse &LU, unsigned LUIdx, const Formula &Base, | |||
3886 | const SmallVectorImpl<int64_t> &Worklist, size_t Idx, bool IsScaledReg) { | |||
3887 | ||||
3888 | auto GenerateOffset = [&](const SCEV *G, int64_t Offset) { | |||
3889 | Formula F = Base; | |||
3890 | F.BaseOffset = (uint64_t)Base.BaseOffset - Offset; | |||
3891 | ||||
3892 | if (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) { | |||
3893 | // Add the offset to the base register. | |||
3894 | const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), Offset), G); | |||
3895 | // If it cancelled out, drop the base register, otherwise update it. | |||
3896 | if (NewG->isZero()) { | |||
3897 | if (IsScaledReg) { | |||
3898 | F.Scale = 0; | |||
3899 | F.ScaledReg = nullptr; | |||
3900 | } else | |||
3901 | F.deleteBaseReg(F.BaseRegs[Idx]); | |||
3902 | F.canonicalize(*L); | |||
3903 | } else if (IsScaledReg) | |||
3904 | F.ScaledReg = NewG; | |||
3905 | else | |||
3906 | F.BaseRegs[Idx] = NewG; | |||
3907 | ||||
3908 | (void)InsertFormula(LU, LUIdx, F); | |||
3909 | } | |||
3910 | }; | |||
3911 | ||||
3912 | const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx]; | |||
3913 | ||||
3914 | // With constant offsets and constant steps, we can generate pre-inc | |||
3915 | // accesses by having the offset equal the step. So, for access #0 with a | |||
3916 | // step of 8, we generate a G - 8 base which would require the first access | |||
3917 | // to be ((G - 8) + 8),+,8. The pre-indexed access then updates the pointer | |||
3918 | // for itself and hopefully becomes the base for other accesses. This means | |||
3919 | // means that a single pre-indexed access can be generated to become the new | |||
3920 | // base pointer for each iteration of the loop, resulting in no extra add/sub | |||
3921 | // instructions for pointer updating. | |||
3922 | if (AMK == TTI::AMK_PreIndexed && LU.Kind == LSRUse::Address) { | |||
3923 | if (auto *GAR = dyn_cast<SCEVAddRecExpr>(G)) { | |||
3924 | if (auto *StepRec = | |||
3925 | dyn_cast<SCEVConstant>(GAR->getStepRecurrence(SE))) { | |||
3926 | const APInt &StepInt = StepRec->getAPInt(); | |||
3927 | int64_t Step = StepInt.isNegative() ? | |||
3928 | StepInt.getSExtValue() : StepInt.getZExtValue(); | |||
3929 | ||||
3930 | for (int64_t Offset : Worklist) { | |||
3931 | Offset -= Step; | |||
3932 | GenerateOffset(G, Offset); | |||
3933 | } | |||
3934 | } | |||
3935 | } | |||
3936 | } | |||
3937 | for (int64_t Offset : Worklist) | |||
3938 | GenerateOffset(G, Offset); | |||
3939 | ||||
3940 | int64_t Imm = ExtractImmediate(G, SE); | |||
3941 | if (G->isZero() || Imm == 0) | |||
3942 | return; | |||
3943 | Formula F = Base; | |||
3944 | F.BaseOffset = (uint64_t)F.BaseOffset + Imm; | |||
3945 | if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) | |||
3946 | return; | |||
3947 | if (IsScaledReg) { | |||
3948 | F.ScaledReg = G; | |||
3949 | } else { | |||
3950 | F.BaseRegs[Idx] = G; | |||
3951 | // We may generate non canonical Formula if G is a recurrent expr reg | |||
3952 | // related with current loop while F.ScaledReg is not. | |||
3953 | F.canonicalize(*L); | |||
3954 | } | |||
3955 | (void)InsertFormula(LU, LUIdx, F); | |||
3956 | } | |||
3957 | ||||
3958 | /// GenerateConstantOffsets - Generate reuse formulae using symbolic offsets. | |||
3959 | void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, | |||
3960 | Formula Base) { | |||
3961 | // TODO: For now, just add the min and max offset, because it usually isn't | |||
3962 | // worthwhile looking at everything inbetween. | |||
3963 | SmallVector<int64_t, 2> Worklist; | |||
3964 | Worklist.push_back(LU.MinOffset); | |||
3965 | if (LU.MaxOffset != LU.MinOffset) | |||
3966 | Worklist.push_back(LU.MaxOffset); | |||
3967 | ||||
3968 | for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) | |||
3969 | GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, i); | |||
3970 | if (Base.Scale == 1) | |||
3971 | GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1, | |||
3972 | /* IsScaledReg */ true); | |||
3973 | } | |||
3974 | ||||
3975 | /// For ICmpZero, check to see if we can scale up the comparison. For example, x | |||
3976 | /// == y -> x*c == y*c. | |||
3977 | void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, | |||
3978 | Formula Base) { | |||
3979 | if (LU.Kind != LSRUse::ICmpZero) return; | |||
3980 | ||||
3981 | // Determine the integer type for the base formula. | |||
3982 | Type *IntTy = Base.getType(); | |||
3983 | if (!IntTy) return; | |||
3984 | if (SE.getTypeSizeInBits(IntTy) > 64) return; | |||
3985 | ||||
3986 | // Don't do this if there is more than one offset. | |||
3987 | if (LU.MinOffset != LU.MaxOffset) return; | |||
3988 | ||||
3989 | // Check if transformation is valid. It is illegal to multiply pointer. | |||
3990 | if (Base.ScaledReg && Base.ScaledReg->getType()->isPointerTy()) | |||
3991 | return; | |||
3992 | for (const SCEV *BaseReg : Base.BaseRegs) | |||
3993 | if (BaseReg->getType()->isPointerTy()) | |||
3994 | return; | |||
3995 | assert(!Base.BaseGV && "ICmpZero use is not legal!")(static_cast <bool> (!Base.BaseGV && "ICmpZero use is not legal!" ) ? void (0) : __assert_fail ("!Base.BaseGV && \"ICmpZero use is not legal!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 3995, __extension__ __PRETTY_FUNCTION__)); | |||
3996 | ||||
3997 | // Check each interesting stride. | |||
3998 | for (int64_t Factor : Factors) { | |||
3999 | // Check that Factor can be represented by IntTy | |||
4000 | if (!ConstantInt::isValueValidForType(IntTy, Factor)) | |||
4001 | continue; | |||
4002 | // Check that the multiplication doesn't overflow. | |||
4003 | if (Base.BaseOffset == std::numeric_limits<int64_t>::min() && Factor == -1) | |||
4004 | continue; | |||
4005 | int64_t NewBaseOffset = (uint64_t)Base.BaseOffset * Factor; | |||
4006 | assert(Factor != 0 && "Zero factor not expected!")(static_cast <bool> (Factor != 0 && "Zero factor not expected!" ) ? void (0) : __assert_fail ("Factor != 0 && \"Zero factor not expected!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 4006, __extension__ __PRETTY_FUNCTION__)); | |||
4007 | if (NewBaseOffset / Factor != Base.BaseOffset) | |||
4008 | continue; | |||
4009 | // If the offset will be truncated at this use, check that it is in bounds. | |||
4010 | if (!IntTy->isPointerTy() && | |||
4011 | !ConstantInt::isValueValidForType(IntTy, NewBaseOffset)) | |||
4012 | continue; | |||
4013 | ||||
4014 | // Check that multiplying with the use offset doesn't overflow. | |||
4015 | int64_t Offset = LU.MinOffset; | |||
4016 | if (Offset == std::numeric_limits<int64_t>::min() && Factor == -1) | |||
4017 | continue; | |||
4018 | Offset = (uint64_t)Offset * Factor; | |||
4019 | if (Offset / Factor != LU.MinOffset) | |||
4020 | continue; | |||
4021 | // If the offset will be truncated at this use, check that it is in bounds. | |||
4022 | if (!IntTy->isPointerTy() && | |||
4023 | !ConstantInt::isValueValidForType(IntTy, Offset)) | |||
4024 | continue; | |||
4025 | ||||
4026 | Formula F = Base; | |||
4027 | F.BaseOffset = NewBaseOffset; | |||
4028 | ||||
4029 | // Check that this scale is legal. | |||
4030 | if (!isLegalUse(TTI, Offset, Offset, LU.Kind, LU.AccessTy, F)) | |||
4031 | continue; | |||
4032 | ||||
4033 | // Compensate for the use having MinOffset built into it. | |||
4034 | F.BaseOffset = (uint64_t)F.BaseOffset + Offset - LU.MinOffset; | |||
4035 | ||||
4036 | const SCEV *FactorS = SE.getConstant(IntTy, Factor); | |||
4037 | ||||
4038 | // Check that multiplying with each base register doesn't overflow. | |||
4039 | for (size_t i = 0, e = F.BaseRegs.size(); i != e; ++i) { | |||
4040 | F.BaseRegs[i] = SE.getMulExpr(F.BaseRegs[i], FactorS); | |||
4041 | if (getExactSDiv(F.BaseRegs[i], FactorS, SE) != Base.BaseRegs[i]) | |||
4042 | goto next; | |||
4043 | } | |||
4044 | ||||
4045 | // Check that multiplying with the scaled register doesn't overflow. | |||
4046 | if (F.ScaledReg) { | |||
4047 | F.ScaledReg = SE.getMulExpr(F.ScaledReg, FactorS); | |||
4048 | if (getExactSDiv(F.ScaledReg, FactorS, SE) != Base.ScaledReg) | |||
4049 | continue; | |||
4050 | } | |||
4051 | ||||
4052 | // Check that multiplying with the unfolded offset doesn't overflow. | |||
4053 | if (F.UnfoldedOffset != 0) { | |||
4054 | if (F.UnfoldedOffset == std::numeric_limits<int64_t>::min() && | |||
4055 | Factor == -1) | |||
4056 | continue; | |||
4057 | F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset * Factor; | |||
4058 | if (F.UnfoldedOffset / Factor != Base.UnfoldedOffset) | |||
4059 | continue; | |||
4060 | // If the offset will be truncated, check that it is in bounds. | |||
4061 | if (!IntTy->isPointerTy() && | |||
4062 | !ConstantInt::isValueValidForType(IntTy, F.UnfoldedOffset)) | |||
4063 | continue; | |||
4064 | } | |||
4065 | ||||
4066 | // If we make it here and it's legal, add it. | |||
4067 | (void)InsertFormula(LU, LUIdx, F); | |||
4068 | next:; | |||
4069 | } | |||
4070 | } | |||
4071 | ||||
4072 | /// Generate stride factor reuse formulae by making use of scaled-offset address | |||
4073 | /// modes, for example. | |||
4074 | void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) { | |||
4075 | // Determine the integer type for the base formula. | |||
4076 | Type *IntTy = Base.getType(); | |||
4077 | if (!IntTy) return; | |||
4078 | ||||
4079 | // If this Formula already has a scaled register, we can't add another one. | |||
4080 | // Try to unscale the formula to generate a better scale. | |||
4081 | if (Base.Scale != 0 && !Base.unscale()) | |||
4082 | return; | |||
4083 | ||||
4084 | assert(Base.Scale == 0 && "unscale did not did its job!")(static_cast <bool> (Base.Scale == 0 && "unscale did not did its job!" ) ? void (0) : __assert_fail ("Base.Scale == 0 && \"unscale did not did its job!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 4084, __extension__ __PRETTY_FUNCTION__)); | |||
4085 | ||||
4086 | // Check each interesting stride. | |||
4087 | for (int64_t Factor : Factors) { | |||
4088 | Base.Scale = Factor; | |||
4089 | Base.HasBaseReg = Base.BaseRegs.size() > 1; | |||
4090 | // Check whether this scale is going to be legal. | |||
4091 | if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, | |||
4092 | Base)) { | |||
4093 | // As a special-case, handle special out-of-loop Basic users specially. | |||
4094 | // TODO: Reconsider this special case. | |||
4095 | if (LU.Kind == LSRUse::Basic && | |||
4096 | isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LSRUse::Special, | |||
4097 | LU.AccessTy, Base) && | |||
4098 | LU.AllFixupsOutsideLoop) | |||
4099 | LU.Kind = LSRUse::Special; | |||
4100 | else | |||
4101 | continue; | |||
4102 | } | |||
4103 | // For an ICmpZero, negating a solitary base register won't lead to | |||
4104 | // new solutions. | |||
4105 | if (LU.Kind == LSRUse::ICmpZero && | |||
4106 | !Base.HasBaseReg && Base.BaseOffset == 0 && !Base.BaseGV) | |||
4107 | continue; | |||
4108 | // For each addrec base reg, if its loop is current loop, apply the scale. | |||
4109 | for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) { | |||
4110 | const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Base.BaseRegs[i]); | |||
4111 | if (AR && (AR->getLoop() == L || LU.AllFixupsOutsideLoop)) { | |||
4112 | const SCEV *FactorS = SE.getConstant(IntTy, Factor); | |||
4113 | if (FactorS->isZero()) | |||
4114 | continue; | |||
4115 | // Divide out the factor, ignoring high bits, since we'll be | |||
4116 | // scaling the value back up in the end. | |||
4117 | if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true)) | |||
4118 | if (!Quotient->isZero()) { | |||
4119 | // TODO: This could be optimized to avoid all the copying. | |||
4120 | Formula F = Base; | |||
4121 | F.ScaledReg = Quotient; | |||
4122 | F.deleteBaseReg(F.BaseRegs[i]); | |||
4123 | // The canonical representation of 1*reg is reg, which is already in | |||
4124 | // Base. In that case, do not try to insert the formula, it will be | |||
4125 | // rejected anyway. | |||
4126 | if (F.Scale == 1 && (F.BaseRegs.empty() || | |||
4127 | (AR->getLoop() != L && LU.AllFixupsOutsideLoop))) | |||
4128 | continue; | |||
4129 | // If AllFixupsOutsideLoop is true and F.Scale is 1, we may generate | |||
4130 | // non canonical Formula with ScaledReg's loop not being L. | |||
4131 | if (F.Scale == 1 && LU.AllFixupsOutsideLoop) | |||
4132 | F.canonicalize(*L); | |||
4133 | (void)InsertFormula(LU, LUIdx, F); | |||
4134 | } | |||
4135 | } | |||
4136 | } | |||
4137 | } | |||
4138 | } | |||
4139 | ||||
4140 | /// Generate reuse formulae from different IV types. | |||
4141 | void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) { | |||
4142 | // Don't bother truncating symbolic values. | |||
4143 | if (Base.BaseGV) return; | |||
4144 | ||||
4145 | // Determine the integer type for the base formula. | |||
4146 | Type *DstTy = Base.getType(); | |||
4147 | if (!DstTy) return; | |||
4148 | if (DstTy->isPointerTy()) | |||
4149 | return; | |||
4150 | ||||
4151 | // It is invalid to extend a pointer type so exit early if ScaledReg or | |||
4152 | // any of the BaseRegs are pointers. | |||
4153 | if (Base.ScaledReg && Base.ScaledReg->getType()->isPointerTy()) | |||
4154 | return; | |||
4155 | if (any_of(Base.BaseRegs, | |||
4156 | [](const SCEV *S) { return S->getType()->isPointerTy(); })) | |||
4157 | return; | |||
4158 | ||||
4159 | for (Type *SrcTy : Types) { | |||
4160 | if (SrcTy != DstTy && TTI.isTruncateFree(SrcTy, DstTy)) { | |||
4161 | Formula F = Base; | |||
4162 | ||||
4163 | // Sometimes SCEV is able to prove zero during ext transform. It may | |||
4164 | // happen if SCEV did not do all possible transforms while creating the | |||
4165 | // initial node (maybe due to depth limitations), but it can do them while | |||
4166 | // taking ext. | |||
4167 | if (F.ScaledReg) { | |||
4168 | const SCEV *NewScaledReg = SE.getAnyExtendExpr(F.ScaledReg, SrcTy); | |||
4169 | if (NewScaledReg->isZero()) | |||
4170 | continue; | |||
4171 | F.ScaledReg = NewScaledReg; | |||
4172 | } | |||
4173 | bool HasZeroBaseReg = false; | |||
4174 | for (const SCEV *&BaseReg : F.BaseRegs) { | |||
4175 | const SCEV *NewBaseReg = SE.getAnyExtendExpr(BaseReg, SrcTy); | |||
4176 | if (NewBaseReg->isZero()) { | |||
4177 | HasZeroBaseReg = true; | |||
4178 | break; | |||
4179 | } | |||
4180 | BaseReg = NewBaseReg; | |||
4181 | } | |||
4182 | if (HasZeroBaseReg) | |||
4183 | continue; | |||
4184 | ||||
4185 | // TODO: This assumes we've done basic processing on all uses and | |||
4186 | // have an idea what the register usage is. | |||
4187 | if (!F.hasRegsUsedByUsesOtherThan(LUIdx, RegUses)) | |||
4188 | continue; | |||
4189 | ||||
4190 | F.canonicalize(*L); | |||
4191 | (void)InsertFormula(LU, LUIdx, F); | |||
4192 | } | |||
4193 | } | |||
4194 | } | |||
4195 | ||||
4196 | namespace { | |||
4197 | ||||
4198 | /// Helper class for GenerateCrossUseConstantOffsets. It's used to defer | |||
4199 | /// modifications so that the search phase doesn't have to worry about the data | |||
4200 | /// structures moving underneath it. | |||
4201 | struct WorkItem { | |||
4202 | size_t LUIdx; | |||
4203 | int64_t Imm; | |||
4204 | const SCEV *OrigReg; | |||
4205 | ||||
4206 | WorkItem(size_t LI, int64_t I, const SCEV *R) | |||
4207 | : LUIdx(LI), Imm(I), OrigReg(R) {} | |||
4208 | ||||
4209 | void print(raw_ostream &OS) const; | |||
4210 | void dump() const; | |||
4211 | }; | |||
4212 | ||||
4213 | } // end anonymous namespace | |||
4214 | ||||
4215 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
4216 | void WorkItem::print(raw_ostream &OS) const { | |||
4217 | OS << "in formulae referencing " << *OrigReg << " in use " << LUIdx | |||
4218 | << " , add offset " << Imm; | |||
4219 | } | |||
4220 | ||||
4221 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void WorkItem::dump() const { | |||
4222 | print(errs()); errs() << '\n'; | |||
4223 | } | |||
4224 | #endif | |||
4225 | ||||
4226 | /// Look for registers which are a constant distance apart and try to form reuse | |||
4227 | /// opportunities between them. | |||
4228 | void LSRInstance::GenerateCrossUseConstantOffsets() { | |||
4229 | // Group the registers by their value without any added constant offset. | |||
4230 | using ImmMapTy = std::map<int64_t, const SCEV *>; | |||
4231 | ||||
4232 | DenseMap<const SCEV *, ImmMapTy> Map; | |||
4233 | DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap; | |||
4234 | SmallVector<const SCEV *, 8> Sequence; | |||
4235 | for (const SCEV *Use : RegUses) { | |||
4236 | const SCEV *Reg = Use; // Make a copy for ExtractImmediate to modify. | |||
4237 | int64_t Imm = ExtractImmediate(Reg, SE); | |||
4238 | auto Pair = Map.insert(std::make_pair(Reg, ImmMapTy())); | |||
4239 | if (Pair.second) | |||
4240 | Sequence.push_back(Reg); | |||
4241 | Pair.first->second.insert(std::make_pair(Imm, Use)); | |||
4242 | UsedByIndicesMap[Reg] |= RegUses.getUsedByIndices(Use); | |||
4243 | } | |||
4244 | ||||
4245 | // Now examine each set of registers with the same base value. Build up | |||
4246 | // a list of work to do and do the work in a separate step so that we're | |||
4247 | // not adding formulae and register counts while we're searching. | |||
4248 | SmallVector<WorkItem, 32> WorkItems; | |||
4249 | SmallSet<std::pair<size_t, int64_t>, 32> UniqueItems; | |||
4250 | for (const SCEV *Reg : Sequence) { | |||
4251 | const ImmMapTy &Imms = Map.find(Reg)->second; | |||
4252 | ||||
4253 | // It's not worthwhile looking for reuse if there's only one offset. | |||
4254 | if (Imms.size() == 1) | |||
4255 | continue; | |||
4256 | ||||
4257 | LLVM_DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << ':';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Generating cross-use offsets for " << *Reg << ':'; for (const auto &Entry : Imms ) dbgs() << ' ' << Entry.first; dbgs() << '\n' ; } } while (false) | |||
4258 | for (const auto &Entrydo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Generating cross-use offsets for " << *Reg << ':'; for (const auto &Entry : Imms ) dbgs() << ' ' << Entry.first; dbgs() << '\n' ; } } while (false) | |||
4259 | : Imms) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Generating cross-use offsets for " << *Reg << ':'; for (const auto &Entry : Imms ) dbgs() << ' ' << Entry.first; dbgs() << '\n' ; } } while (false) | |||
4260 | << ' ' << Entry.first;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Generating cross-use offsets for " << *Reg << ':'; for (const auto &Entry : Imms ) dbgs() << ' ' << Entry.first; dbgs() << '\n' ; } } while (false) | |||
4261 | dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Generating cross-use offsets for " << *Reg << ':'; for (const auto &Entry : Imms ) dbgs() << ' ' << Entry.first; dbgs() << '\n' ; } } while (false); | |||
4262 | ||||
4263 | // Examine each offset. | |||
4264 | for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end(); | |||
4265 | J != JE; ++J) { | |||
4266 | const SCEV *OrigReg = J->second; | |||
4267 | ||||
4268 | int64_t JImm = J->first; | |||
4269 | const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(OrigReg); | |||
4270 | ||||
4271 | if (!isa<SCEVConstant>(OrigReg) && | |||
4272 | UsedByIndicesMap[Reg].count() == 1) { | |||
4273 | LLVM_DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigRegdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Skipping cross-use reuse for " << *OrigReg << '\n'; } } while (false) | |||
4274 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Skipping cross-use reuse for " << *OrigReg << '\n'; } } while (false); | |||
4275 | continue; | |||
4276 | } | |||
4277 | ||||
4278 | // Conservatively examine offsets between this orig reg a few selected | |||
4279 | // other orig regs. | |||
4280 | int64_t First = Imms.begin()->first; | |||
4281 | int64_t Last = std::prev(Imms.end())->first; | |||
4282 | // Compute (First + Last) / 2 without overflow using the fact that | |||
4283 | // First + Last = 2 * (First + Last) + (First ^ Last). | |||
4284 | int64_t Avg = (First & Last) + ((First ^ Last) >> 1); | |||
4285 | // If the result is negative and First is odd and Last even (or vice versa), | |||
4286 | // we rounded towards -inf. Add 1 in that case, to round towards 0. | |||
4287 | Avg = Avg + ((First ^ Last) & ((uint64_t)Avg >> 63)); | |||
4288 | ImmMapTy::const_iterator OtherImms[] = { | |||
4289 | Imms.begin(), std::prev(Imms.end()), | |||
4290 | Imms.lower_bound(Avg)}; | |||
4291 | for (const auto &M : OtherImms) { | |||
4292 | if (M == J || M == JE) continue; | |||
4293 | ||||
4294 | // Compute the difference between the two. | |||
4295 | int64_t Imm = (uint64_t)JImm - M->first; | |||
4296 | for (unsigned LUIdx : UsedByIndices.set_bits()) | |||
4297 | // Make a memo of this use, offset, and register tuple. | |||
4298 | if (UniqueItems.insert(std::make_pair(LUIdx, Imm)).second) | |||
4299 | WorkItems.push_back(WorkItem(LUIdx, Imm, OrigReg)); | |||
4300 | } | |||
4301 | } | |||
4302 | } | |||
4303 | ||||
4304 | Map.clear(); | |||
4305 | Sequence.clear(); | |||
4306 | UsedByIndicesMap.clear(); | |||
4307 | UniqueItems.clear(); | |||
4308 | ||||
4309 | // Now iterate through the worklist and add new formulae. | |||
4310 | for (const WorkItem &WI : WorkItems) { | |||
4311 | size_t LUIdx = WI.LUIdx; | |||
4312 | LSRUse &LU = Uses[LUIdx]; | |||
4313 | int64_t Imm = WI.Imm; | |||
4314 | const SCEV *OrigReg = WI.OrigReg; | |||
4315 | ||||
4316 | Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType()); | |||
4317 | const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm)); | |||
4318 | unsigned BitWidth = SE.getTypeSizeInBits(IntTy); | |||
4319 | ||||
4320 | // TODO: Use a more targeted data structure. | |||
4321 | for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) { | |||
4322 | Formula F = LU.Formulae[L]; | |||
4323 | // FIXME: The code for the scaled and unscaled registers looks | |||
4324 | // very similar but slightly different. Investigate if they | |||
4325 | // could be merged. That way, we would not have to unscale the | |||
4326 | // Formula. | |||
4327 | F.unscale(); | |||
4328 | // Use the immediate in the scaled register. | |||
4329 | if (F.ScaledReg == OrigReg) { | |||
4330 | int64_t Offset = (uint64_t)F.BaseOffset + Imm * (uint64_t)F.Scale; | |||
4331 | // Don't create 50 + reg(-50). | |||
4332 | if (F.referencesReg(SE.getSCEV( | |||
4333 | ConstantInt::get(IntTy, -(uint64_t)Offset)))) | |||
4334 | continue; | |||
4335 | Formula NewF = F; | |||
4336 | NewF.BaseOffset = Offset; | |||
4337 | if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, | |||
4338 | NewF)) | |||
4339 | continue; | |||
4340 | NewF.ScaledReg = SE.getAddExpr(NegImmS, NewF.ScaledReg); | |||
4341 | ||||
4342 | // If the new scale is a constant in a register, and adding the constant | |||
4343 | // value to the immediate would produce a value closer to zero than the | |||
4344 | // immediate itself, then the formula isn't worthwhile. | |||
4345 | if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewF.ScaledReg)) | |||
4346 | if (C->getValue()->isNegative() != (NewF.BaseOffset < 0) && | |||
4347 | (C->getAPInt().abs() * APInt(BitWidth, F.Scale)) | |||
4348 | .ule(std::abs(NewF.BaseOffset))) | |||
4349 | continue; | |||
4350 | ||||
4351 | // OK, looks good. | |||
4352 | NewF.canonicalize(*this->L); | |||
4353 | (void)InsertFormula(LU, LUIdx, NewF); | |||
4354 | } else { | |||
4355 | // Use the immediate in a base register. | |||
4356 | for (size_t N = 0, NE = F.BaseRegs.size(); N != NE; ++N) { | |||
4357 | const SCEV *BaseReg = F.BaseRegs[N]; | |||
4358 | if (BaseReg != OrigReg) | |||
4359 | continue; | |||
4360 | Formula NewF = F; | |||
4361 | NewF.BaseOffset = (uint64_t)NewF.BaseOffset + Imm; | |||
4362 | if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, | |||
4363 | LU.Kind, LU.AccessTy, NewF)) { | |||
4364 | if (AMK == TTI::AMK_PostIndexed && | |||
4365 | mayUsePostIncMode(TTI, LU, OrigReg, this->L, SE)) | |||
4366 | continue; | |||
4367 | if (!TTI.isLegalAddImmediate((uint64_t)NewF.UnfoldedOffset + Imm)) | |||
4368 | continue; | |||
4369 | NewF = F; | |||
4370 | NewF.UnfoldedOffset = (uint64_t)NewF.UnfoldedOffset + Imm; | |||
4371 | } | |||
4372 | NewF.BaseRegs[N] = SE.getAddExpr(NegImmS, BaseReg); | |||
4373 | ||||
4374 | // If the new formula has a constant in a register, and adding the | |||
4375 | // constant value to the immediate would produce a value closer to | |||
4376 | // zero than the immediate itself, then the formula isn't worthwhile. | |||
4377 | for (const SCEV *NewReg : NewF.BaseRegs) | |||
4378 | if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewReg)) | |||
4379 | if ((C->getAPInt() + NewF.BaseOffset) | |||
4380 | .abs() | |||
4381 | .slt(std::abs(NewF.BaseOffset)) && | |||
4382 | (C->getAPInt() + NewF.BaseOffset).countTrailingZeros() >= | |||
4383 | (unsigned)llvm::countr_zero<uint64_t>(NewF.BaseOffset)) | |||
4384 | goto skip_formula; | |||
4385 | ||||
4386 | // Ok, looks good. | |||
4387 | NewF.canonicalize(*this->L); | |||
4388 | (void)InsertFormula(LU, LUIdx, NewF); | |||
4389 | break; | |||
4390 | skip_formula:; | |||
4391 | } | |||
4392 | } | |||
4393 | } | |||
4394 | } | |||
4395 | } | |||
4396 | ||||
4397 | /// Generate formulae for each use. | |||
4398 | void | |||
4399 | LSRInstance::GenerateAllReuseFormulae() { | |||
4400 | // This is split into multiple loops so that hasRegsUsedByUsesOtherThan | |||
4401 | // queries are more precise. | |||
4402 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { | |||
4403 | LSRUse &LU = Uses[LUIdx]; | |||
4404 | for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) | |||
4405 | GenerateReassociations(LU, LUIdx, LU.Formulae[i]); | |||
4406 | for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) | |||
4407 | GenerateCombinations(LU, LUIdx, LU.Formulae[i]); | |||
4408 | } | |||
4409 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { | |||
4410 | LSRUse &LU = Uses[LUIdx]; | |||
4411 | for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) | |||
4412 | GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]); | |||
4413 | for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) | |||
4414 | GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]); | |||
4415 | for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) | |||
4416 | GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]); | |||
4417 | for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) | |||
4418 | GenerateScales(LU, LUIdx, LU.Formulae[i]); | |||
4419 | } | |||
4420 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { | |||
4421 | LSRUse &LU = Uses[LUIdx]; | |||
4422 | for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i) | |||
4423 | GenerateTruncates(LU, LUIdx, LU.Formulae[i]); | |||
4424 | } | |||
4425 | ||||
4426 | GenerateCrossUseConstantOffsets(); | |||
4427 | ||||
4428 | LLVM_DEBUG(dbgs() << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "After generating reuse formulae:\n" ; print_uses(dbgs()); } } while (false) | |||
4429 | "After generating reuse formulae:\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "After generating reuse formulae:\n" ; print_uses(dbgs()); } } while (false) | |||
4430 | print_uses(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "After generating reuse formulae:\n" ; print_uses(dbgs()); } } while (false); | |||
4431 | } | |||
4432 | ||||
4433 | /// If there are multiple formulae with the same set of registers used | |||
4434 | /// by other uses, pick the best one and delete the others. | |||
4435 | void LSRInstance::FilterOutUndesirableDedicatedRegisters() { | |||
4436 | DenseSet<const SCEV *> VisitedRegs; | |||
4437 | SmallPtrSet<const SCEV *, 16> Regs; | |||
4438 | SmallPtrSet<const SCEV *, 16> LoserRegs; | |||
4439 | #ifndef NDEBUG | |||
4440 | bool ChangedFormulae = false; | |||
4441 | #endif | |||
4442 | ||||
4443 | // Collect the best formula for each unique set of shared registers. This | |||
4444 | // is reset for each use. | |||
4445 | using BestFormulaeTy = | |||
4446 | DenseMap<SmallVector<const SCEV *, 4>, size_t, UniquifierDenseMapInfo>; | |||
4447 | ||||
4448 | BestFormulaeTy BestFormulae; | |||
4449 | ||||
4450 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { | |||
4451 | LSRUse &LU = Uses[LUIdx]; | |||
4452 | LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Filtering for use "; LU.print (dbgs()); dbgs() << '\n'; } } while (false) | |||
4453 | dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Filtering for use "; LU.print (dbgs()); dbgs() << '\n'; } } while (false); | |||
4454 | ||||
4455 | bool Any = false; | |||
4456 | for (size_t FIdx = 0, NumForms = LU.Formulae.size(); | |||
4457 | FIdx != NumForms; ++FIdx) { | |||
4458 | Formula &F = LU.Formulae[FIdx]; | |||
4459 | ||||
4460 | // Some formulas are instant losers. For example, they may depend on | |||
4461 | // nonexistent AddRecs from other loops. These need to be filtered | |||
4462 | // immediately, otherwise heuristics could choose them over others leading | |||
4463 | // to an unsatisfactory solution. Passing LoserRegs into RateFormula here | |||
4464 | // avoids the need to recompute this information across formulae using the | |||
4465 | // same bad AddRec. Passing LoserRegs is also essential unless we remove | |||
4466 | // the corresponding bad register from the Regs set. | |||
4467 | Cost CostF(L, SE, TTI, AMK); | |||
4468 | Regs.clear(); | |||
4469 | CostF.RateFormula(F, Regs, VisitedRegs, LU, &LoserRegs); | |||
4470 | if (CostF.isLoser()) { | |||
4471 | // During initial formula generation, undesirable formulae are generated | |||
4472 | // by uses within other loops that have some non-trivial address mode or | |||
4473 | // use the postinc form of the IV. LSR needs to provide these formulae | |||
4474 | // as the basis of rediscovering the desired formula that uses an AddRec | |||
4475 | // corresponding to the existing phi. Once all formulae have been | |||
4476 | // generated, these initial losers may be pruned. | |||
4477 | LLVM_DEBUG(dbgs() << " Filtering loser "; F.print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Filtering loser "; F.print (dbgs()); dbgs() << "\n"; } } while (false) | |||
4478 | dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Filtering loser "; F.print (dbgs()); dbgs() << "\n"; } } while (false); | |||
4479 | } | |||
4480 | else { | |||
4481 | SmallVector<const SCEV *, 4> Key; | |||
4482 | for (const SCEV *Reg : F.BaseRegs) { | |||
4483 | if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx)) | |||
4484 | Key.push_back(Reg); | |||
4485 | } | |||
4486 | if (F.ScaledReg && | |||
4487 | RegUses.isRegUsedByUsesOtherThan(F.ScaledReg, LUIdx)) | |||
4488 | Key.push_back(F.ScaledReg); | |||
4489 | // Unstable sort by host order ok, because this is only used for | |||
4490 | // uniquifying. | |||
4491 | llvm::sort(Key); | |||
4492 | ||||
4493 | std::pair<BestFormulaeTy::const_iterator, bool> P = | |||
4494 | BestFormulae.insert(std::make_pair(Key, FIdx)); | |||
4495 | if (P.second) | |||
4496 | continue; | |||
4497 | ||||
4498 | Formula &Best = LU.Formulae[P.first->second]; | |||
4499 | ||||
4500 | Cost CostBest(L, SE, TTI, AMK); | |||
4501 | Regs.clear(); | |||
4502 | CostBest.RateFormula(Best, Regs, VisitedRegs, LU); | |||
4503 | if (CostF.isLess(CostBest)) | |||
4504 | std::swap(F, Best); | |||
4505 | LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Filtering out formula " ; F.print(dbgs()); dbgs() << "\n" " in favor of formula " ; Best.print(dbgs()); dbgs() << '\n'; } } while (false) | |||
4506 | dbgs() << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Filtering out formula " ; F.print(dbgs()); dbgs() << "\n" " in favor of formula " ; Best.print(dbgs()); dbgs() << '\n'; } } while (false) | |||
4507 | " in favor of formula ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Filtering out formula " ; F.print(dbgs()); dbgs() << "\n" " in favor of formula " ; Best.print(dbgs()); dbgs() << '\n'; } } while (false) | |||
4508 | Best.print(dbgs()); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Filtering out formula " ; F.print(dbgs()); dbgs() << "\n" " in favor of formula " ; Best.print(dbgs()); dbgs() << '\n'; } } while (false); | |||
4509 | } | |||
4510 | #ifndef NDEBUG | |||
4511 | ChangedFormulae = true; | |||
4512 | #endif | |||
4513 | LU.DeleteFormula(F); | |||
4514 | --FIdx; | |||
4515 | --NumForms; | |||
4516 | Any = true; | |||
4517 | } | |||
4518 | ||||
4519 | // Now that we've filtered out some formulae, recompute the Regs set. | |||
4520 | if (Any) | |||
4521 | LU.RecomputeRegs(LUIdx, RegUses); | |||
4522 | ||||
4523 | // Reset this to prepare for the next use. | |||
4524 | BestFormulae.clear(); | |||
4525 | } | |||
4526 | ||||
4527 | LLVM_DEBUG(if (ChangedFormulae) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ChangedFormulae) { dbgs() << "\n" "After filtering out undesirable candidates:\n"; print_uses( dbgs()); }; } } while (false) | |||
4528 | dbgs() << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ChangedFormulae) { dbgs() << "\n" "After filtering out undesirable candidates:\n"; print_uses( dbgs()); }; } } while (false) | |||
4529 | "After filtering out undesirable candidates:\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ChangedFormulae) { dbgs() << "\n" "After filtering out undesirable candidates:\n"; print_uses( dbgs()); }; } } while (false) | |||
4530 | print_uses(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ChangedFormulae) { dbgs() << "\n" "After filtering out undesirable candidates:\n"; print_uses( dbgs()); }; } } while (false) | |||
4531 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ChangedFormulae) { dbgs() << "\n" "After filtering out undesirable candidates:\n"; print_uses( dbgs()); }; } } while (false); | |||
4532 | } | |||
4533 | ||||
4534 | /// Estimate the worst-case number of solutions the solver might have to | |||
4535 | /// consider. It almost never considers this many solutions because it prune the | |||
4536 | /// search space, but the pruning isn't always sufficient. | |||
4537 | size_t LSRInstance::EstimateSearchSpaceComplexity() const { | |||
4538 | size_t Power = 1; | |||
4539 | for (const LSRUse &LU : Uses) { | |||
4540 | size_t FSize = LU.Formulae.size(); | |||
4541 | if (FSize >= ComplexityLimit) { | |||
4542 | Power = ComplexityLimit; | |||
4543 | break; | |||
4544 | } | |||
4545 | Power *= FSize; | |||
4546 | if (Power >= ComplexityLimit) | |||
4547 | break; | |||
4548 | } | |||
4549 | return Power; | |||
4550 | } | |||
4551 | ||||
4552 | /// When one formula uses a superset of the registers of another formula, it | |||
4553 | /// won't help reduce register pressure (though it may not necessarily hurt | |||
4554 | /// register pressure); remove it to simplify the system. | |||
4555 | void LSRInstance::NarrowSearchSpaceByDetectingSupersets() { | |||
4556 | if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { | |||
4557 | LLVM_DEBUG(dbgs() << "The search space is too complex.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" ; } } while (false); | |||
4558 | ||||
4559 | LLVM_DEBUG(dbgs() << "Narrowing the search space by eliminating formulae "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Narrowing the search space by eliminating formulae " "which use a superset of registers used by other " "formulae.\n" ; } } while (false) | |||
4560 | "which use a superset of registers used by other "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Narrowing the search space by eliminating formulae " "which use a superset of registers used by other " "formulae.\n" ; } } while (false) | |||
4561 | "formulae.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Narrowing the search space by eliminating formulae " "which use a superset of registers used by other " "formulae.\n" ; } } while (false); | |||
4562 | ||||
4563 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { | |||
4564 | LSRUse &LU = Uses[LUIdx]; | |||
4565 | bool Any = false; | |||
4566 | for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { | |||
4567 | Formula &F = LU.Formulae[i]; | |||
4568 | // Look for a formula with a constant or GV in a register. If the use | |||
4569 | // also has a formula with that same value in an immediate field, | |||
4570 | // delete the one that uses a register. | |||
4571 | for (SmallVectorImpl<const SCEV *>::const_iterator | |||
4572 | I = F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) { | |||
4573 | if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*I)) { | |||
4574 | Formula NewF = F; | |||
4575 | //FIXME: Formulas should store bitwidth to do wrapping properly. | |||
4576 | // See PR41034. | |||
4577 | NewF.BaseOffset += (uint64_t)C->getValue()->getSExtValue(); | |||
4578 | NewF.BaseRegs.erase(NewF.BaseRegs.begin() + | |||
4579 | (I - F.BaseRegs.begin())); | |||
4580 | if (LU.HasFormulaWithSameRegs(NewF)) { | |||
4581 | LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Deleting "; F.print(dbgs ()); dbgs() << '\n'; } } while (false) | |||
4582 | dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Deleting "; F.print(dbgs ()); dbgs() << '\n'; } } while (false); | |||
4583 | LU.DeleteFormula(F); | |||
4584 | --i; | |||
4585 | --e; | |||
4586 | Any = true; | |||
4587 | break; | |||
4588 | } | |||
4589 | } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(*I)) { | |||
4590 | if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) | |||
4591 | if (!F.BaseGV) { | |||
4592 | Formula NewF = F; | |||
4593 | NewF.BaseGV = GV; | |||
4594 | NewF.BaseRegs.erase(NewF.BaseRegs.begin() + | |||
4595 | (I - F.BaseRegs.begin())); | |||
4596 | if (LU.HasFormulaWithSameRegs(NewF)) { | |||
4597 | LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Deleting "; F.print(dbgs ()); dbgs() << '\n'; } } while (false) | |||
4598 | dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Deleting "; F.print(dbgs ()); dbgs() << '\n'; } } while (false); | |||
4599 | LU.DeleteFormula(F); | |||
4600 | --i; | |||
4601 | --e; | |||
4602 | Any = true; | |||
4603 | break; | |||
4604 | } | |||
4605 | } | |||
4606 | } | |||
4607 | } | |||
4608 | } | |||
4609 | if (Any) | |||
4610 | LU.RecomputeRegs(LUIdx, RegUses); | |||
4611 | } | |||
4612 | ||||
4613 | LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "After pre-selection:\n"; print_uses (dbgs()); } } while (false); | |||
4614 | } | |||
4615 | } | |||
4616 | ||||
4617 | /// When there are many registers for expressions like A, A+1, A+2, etc., | |||
4618 | /// allocate a single register for them. | |||
4619 | void LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() { | |||
4620 | if (EstimateSearchSpaceComplexity() < ComplexityLimit) | |||
4621 | return; | |||
4622 | ||||
4623 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" "Narrowing the search space by assuming that uses separated " "by a constant offset will use the same registers.\n"; } } while (false) | |||
4624 | dbgs() << "The search space is too complex.\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" "Narrowing the search space by assuming that uses separated " "by a constant offset will use the same registers.\n"; } } while (false) | |||
4625 | "Narrowing the search space by assuming that uses separated "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" "Narrowing the search space by assuming that uses separated " "by a constant offset will use the same registers.\n"; } } while (false) | |||
4626 | "by a constant offset will use the same registers.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" "Narrowing the search space by assuming that uses separated " "by a constant offset will use the same registers.\n"; } } while (false); | |||
4627 | ||||
4628 | // This is especially useful for unrolled loops. | |||
4629 | ||||
4630 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { | |||
4631 | LSRUse &LU = Uses[LUIdx]; | |||
4632 | for (const Formula &F : LU.Formulae) { | |||
4633 | if (F.BaseOffset == 0 || (F.Scale != 0 && F.Scale != 1)) | |||
4634 | continue; | |||
4635 | ||||
4636 | LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU); | |||
4637 | if (!LUThatHas) | |||
4638 | continue; | |||
4639 | ||||
4640 | if (!reconcileNewOffset(*LUThatHas, F.BaseOffset, /*HasBaseReg=*/ false, | |||
4641 | LU.Kind, LU.AccessTy)) | |||
4642 | continue; | |||
4643 | ||||
4644 | LLVM_DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Deleting use "; LU.print (dbgs()); dbgs() << '\n'; } } while (false); | |||
4645 | ||||
4646 | LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop; | |||
4647 | ||||
4648 | // Transfer the fixups of LU to LUThatHas. | |||
4649 | for (LSRFixup &Fixup : LU.Fixups) { | |||
4650 | Fixup.Offset += F.BaseOffset; | |||
4651 | LUThatHas->pushFixup(Fixup); | |||
4652 | LLVM_DEBUG(dbgs() << "New fixup has offset " << Fixup.Offset << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "New fixup has offset " << Fixup.Offset << '\n'; } } while (false); | |||
4653 | } | |||
4654 | ||||
4655 | // Delete formulae from the new use which are no longer legal. | |||
4656 | bool Any = false; | |||
4657 | for (size_t i = 0, e = LUThatHas->Formulae.size(); i != e; ++i) { | |||
4658 | Formula &F = LUThatHas->Formulae[i]; | |||
4659 | if (!isLegalUse(TTI, LUThatHas->MinOffset, LUThatHas->MaxOffset, | |||
4660 | LUThatHas->Kind, LUThatHas->AccessTy, F)) { | |||
4661 | LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Deleting "; F.print(dbgs ()); dbgs() << '\n'; } } while (false); | |||
4662 | LUThatHas->DeleteFormula(F); | |||
4663 | --i; | |||
4664 | --e; | |||
4665 | Any = true; | |||
4666 | } | |||
4667 | } | |||
4668 | ||||
4669 | if (Any) | |||
4670 | LUThatHas->RecomputeRegs(LUThatHas - &Uses.front(), RegUses); | |||
4671 | ||||
4672 | // Delete the old use. | |||
4673 | DeleteUse(LU, LUIdx); | |||
4674 | --LUIdx; | |||
4675 | --NumUses; | |||
4676 | break; | |||
4677 | } | |||
4678 | } | |||
4679 | ||||
4680 | LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "After pre-selection:\n"; print_uses (dbgs()); } } while (false); | |||
4681 | } | |||
4682 | ||||
4683 | /// Call FilterOutUndesirableDedicatedRegisters again, if necessary, now that | |||
4684 | /// we've done more filtering, as it may be able to find more formulae to | |||
4685 | /// eliminate. | |||
4686 | void LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){ | |||
4687 | if (EstimateSearchSpaceComplexity() >= ComplexityLimit) { | |||
4688 | LLVM_DEBUG(dbgs() << "The search space is too complex.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" ; } } while (false); | |||
4689 | ||||
4690 | LLVM_DEBUG(dbgs() << "Narrowing the search space by re-filtering out "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Narrowing the search space by re-filtering out " "undesirable dedicated registers.\n"; } } while (false) | |||
4691 | "undesirable dedicated registers.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Narrowing the search space by re-filtering out " "undesirable dedicated registers.\n"; } } while (false); | |||
4692 | ||||
4693 | FilterOutUndesirableDedicatedRegisters(); | |||
4694 | ||||
4695 | LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "After pre-selection:\n"; print_uses (dbgs()); } } while (false); | |||
4696 | } | |||
4697 | } | |||
4698 | ||||
4699 | /// If a LSRUse has multiple formulae with the same ScaledReg and Scale. | |||
4700 | /// Pick the best one and delete the others. | |||
4701 | /// This narrowing heuristic is to keep as many formulae with different | |||
4702 | /// Scale and ScaledReg pair as possible while narrowing the search space. | |||
4703 | /// The benefit is that it is more likely to find out a better solution | |||
4704 | /// from a formulae set with more Scale and ScaledReg variations than | |||
4705 | /// a formulae set with the same Scale and ScaledReg. The picking winner | |||
4706 | /// reg heuristic will often keep the formulae with the same Scale and | |||
4707 | /// ScaledReg and filter others, and we want to avoid that if possible. | |||
4708 | void LSRInstance::NarrowSearchSpaceByFilterFormulaWithSameScaledReg() { | |||
4709 | if (EstimateSearchSpaceComplexity() < ComplexityLimit) | |||
4710 | return; | |||
4711 | ||||
4712 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" "Narrowing the search space by choosing the best Formula " "from the Formulae with the same Scale and ScaledReg.\n" ; } } while (false) | |||
4713 | dbgs() << "The search space is too complex.\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" "Narrowing the search space by choosing the best Formula " "from the Formulae with the same Scale and ScaledReg.\n" ; } } while (false) | |||
4714 | "Narrowing the search space by choosing the best Formula "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" "Narrowing the search space by choosing the best Formula " "from the Formulae with the same Scale and ScaledReg.\n" ; } } while (false) | |||
4715 | "from the Formulae with the same Scale and ScaledReg.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" "Narrowing the search space by choosing the best Formula " "from the Formulae with the same Scale and ScaledReg.\n" ; } } while (false); | |||
4716 | ||||
4717 | // Map the "Scale * ScaledReg" pair to the best formula of current LSRUse. | |||
4718 | using BestFormulaeTy = DenseMap<std::pair<const SCEV *, int64_t>, size_t>; | |||
4719 | ||||
4720 | BestFormulaeTy BestFormulae; | |||
4721 | #ifndef NDEBUG | |||
4722 | bool ChangedFormulae = false; | |||
4723 | #endif | |||
4724 | DenseSet<const SCEV *> VisitedRegs; | |||
4725 | SmallPtrSet<const SCEV *, 16> Regs; | |||
4726 | ||||
4727 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { | |||
4728 | LSRUse &LU = Uses[LUIdx]; | |||
4729 | LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Filtering for use "; LU.print (dbgs()); dbgs() << '\n'; } } while (false) | |||
4730 | dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Filtering for use "; LU.print (dbgs()); dbgs() << '\n'; } } while (false); | |||
4731 | ||||
4732 | // Return true if Formula FA is better than Formula FB. | |||
4733 | auto IsBetterThan = [&](Formula &FA, Formula &FB) { | |||
4734 | // First we will try to choose the Formula with fewer new registers. | |||
4735 | // For a register used by current Formula, the more the register is | |||
4736 | // shared among LSRUses, the less we increase the register number | |||
4737 | // counter of the formula. | |||
4738 | size_t FARegNum = 0; | |||
4739 | for (const SCEV *Reg : FA.BaseRegs) { | |||
4740 | const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg); | |||
4741 | FARegNum += (NumUses - UsedByIndices.count() + 1); | |||
4742 | } | |||
4743 | size_t FBRegNum = 0; | |||
4744 | for (const SCEV *Reg : FB.BaseRegs) { | |||
4745 | const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg); | |||
4746 | FBRegNum += (NumUses - UsedByIndices.count() + 1); | |||
4747 | } | |||
4748 | if (FARegNum != FBRegNum) | |||
4749 | return FARegNum < FBRegNum; | |||
4750 | ||||
4751 | // If the new register numbers are the same, choose the Formula with | |||
4752 | // less Cost. | |||
4753 | Cost CostFA(L, SE, TTI, AMK); | |||
4754 | Cost CostFB(L, SE, TTI, AMK); | |||
4755 | Regs.clear(); | |||
4756 | CostFA.RateFormula(FA, Regs, VisitedRegs, LU); | |||
4757 | Regs.clear(); | |||
4758 | CostFB.RateFormula(FB, Regs, VisitedRegs, LU); | |||
4759 | return CostFA.isLess(CostFB); | |||
4760 | }; | |||
4761 | ||||
4762 | bool Any = false; | |||
4763 | for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms; | |||
4764 | ++FIdx) { | |||
4765 | Formula &F = LU.Formulae[FIdx]; | |||
4766 | if (!F.ScaledReg) | |||
4767 | continue; | |||
4768 | auto P = BestFormulae.insert({{F.ScaledReg, F.Scale}, FIdx}); | |||
4769 | if (P.second) | |||
4770 | continue; | |||
4771 | ||||
4772 | Formula &Best = LU.Formulae[P.first->second]; | |||
4773 | if (IsBetterThan(F, Best)) | |||
4774 | std::swap(F, Best); | |||
4775 | LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Filtering out formula " ; F.print(dbgs()); dbgs() << "\n" " in favor of formula " ; Best.print(dbgs()); dbgs() << '\n'; } } while (false) | |||
4776 | dbgs() << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Filtering out formula " ; F.print(dbgs()); dbgs() << "\n" " in favor of formula " ; Best.print(dbgs()); dbgs() << '\n'; } } while (false) | |||
4777 | " in favor of formula ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Filtering out formula " ; F.print(dbgs()); dbgs() << "\n" " in favor of formula " ; Best.print(dbgs()); dbgs() << '\n'; } } while (false) | |||
4778 | Best.print(dbgs()); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Filtering out formula " ; F.print(dbgs()); dbgs() << "\n" " in favor of formula " ; Best.print(dbgs()); dbgs() << '\n'; } } while (false); | |||
4779 | #ifndef NDEBUG | |||
4780 | ChangedFormulae = true; | |||
4781 | #endif | |||
4782 | LU.DeleteFormula(F); | |||
4783 | --FIdx; | |||
4784 | --NumForms; | |||
4785 | Any = true; | |||
4786 | } | |||
4787 | if (Any) | |||
4788 | LU.RecomputeRegs(LUIdx, RegUses); | |||
4789 | ||||
4790 | // Reset this to prepare for the next use. | |||
4791 | BestFormulae.clear(); | |||
4792 | } | |||
4793 | ||||
4794 | LLVM_DEBUG(if (ChangedFormulae) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ChangedFormulae) { dbgs() << "\n" "After filtering out undesirable candidates:\n"; print_uses( dbgs()); }; } } while (false) | |||
4795 | dbgs() << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ChangedFormulae) { dbgs() << "\n" "After filtering out undesirable candidates:\n"; print_uses( dbgs()); }; } } while (false) | |||
4796 | "After filtering out undesirable candidates:\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ChangedFormulae) { dbgs() << "\n" "After filtering out undesirable candidates:\n"; print_uses( dbgs()); }; } } while (false) | |||
4797 | print_uses(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ChangedFormulae) { dbgs() << "\n" "After filtering out undesirable candidates:\n"; print_uses( dbgs()); }; } } while (false) | |||
4798 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ChangedFormulae) { dbgs() << "\n" "After filtering out undesirable candidates:\n"; print_uses( dbgs()); }; } } while (false); | |||
4799 | } | |||
4800 | ||||
4801 | /// If we are over the complexity limit, filter out any post-inc prefering | |||
4802 | /// variables to only post-inc values. | |||
4803 | void LSRInstance::NarrowSearchSpaceByFilterPostInc() { | |||
4804 | if (AMK != TTI::AMK_PostIndexed) | |||
4805 | return; | |||
4806 | if (EstimateSearchSpaceComplexity() < ComplexityLimit) | |||
4807 | return; | |||
4808 | ||||
4809 | LLVM_DEBUG(dbgs() << "The search space is too complex.\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" "Narrowing the search space by choosing the lowest " "register Formula for PostInc Uses.\n" ; } } while (false) | |||
4810 | "Narrowing the search space by choosing the lowest "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" "Narrowing the search space by choosing the lowest " "register Formula for PostInc Uses.\n" ; } } while (false) | |||
4811 | "register Formula for PostInc Uses.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" "Narrowing the search space by choosing the lowest " "register Formula for PostInc Uses.\n" ; } } while (false); | |||
4812 | ||||
4813 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { | |||
4814 | LSRUse &LU = Uses[LUIdx]; | |||
4815 | ||||
4816 | if (LU.Kind != LSRUse::Address) | |||
4817 | continue; | |||
4818 | if (!TTI.isIndexedLoadLegal(TTI.MIM_PostInc, LU.AccessTy.getType()) && | |||
4819 | !TTI.isIndexedStoreLegal(TTI.MIM_PostInc, LU.AccessTy.getType())) | |||
4820 | continue; | |||
4821 | ||||
4822 | size_t MinRegs = std::numeric_limits<size_t>::max(); | |||
4823 | for (const Formula &F : LU.Formulae) | |||
4824 | MinRegs = std::min(F.getNumRegs(), MinRegs); | |||
4825 | ||||
4826 | bool Any = false; | |||
4827 | for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms; | |||
4828 | ++FIdx) { | |||
4829 | Formula &F = LU.Formulae[FIdx]; | |||
4830 | if (F.getNumRegs() > MinRegs) { | |||
4831 | LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Filtering out formula " ; F.print(dbgs()); dbgs() << "\n"; } } while (false) | |||
4832 | dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Filtering out formula " ; F.print(dbgs()); dbgs() << "\n"; } } while (false); | |||
4833 | LU.DeleteFormula(F); | |||
4834 | --FIdx; | |||
4835 | --NumForms; | |||
4836 | Any = true; | |||
4837 | } | |||
4838 | } | |||
4839 | if (Any) | |||
4840 | LU.RecomputeRegs(LUIdx, RegUses); | |||
4841 | ||||
4842 | if (EstimateSearchSpaceComplexity() < ComplexityLimit) | |||
4843 | break; | |||
4844 | } | |||
4845 | ||||
4846 | LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "After pre-selection:\n"; print_uses (dbgs()); } } while (false); | |||
4847 | } | |||
4848 | ||||
4849 | /// The function delete formulas with high registers number expectation. | |||
4850 | /// Assuming we don't know the value of each formula (already delete | |||
4851 | /// all inefficient), generate probability of not selecting for each | |||
4852 | /// register. | |||
4853 | /// For example, | |||
4854 | /// Use1: | |||
4855 | /// reg(a) + reg({0,+,1}) | |||
4856 | /// reg(a) + reg({-1,+,1}) + 1 | |||
4857 | /// reg({a,+,1}) | |||
4858 | /// Use2: | |||
4859 | /// reg(b) + reg({0,+,1}) | |||
4860 | /// reg(b) + reg({-1,+,1}) + 1 | |||
4861 | /// reg({b,+,1}) | |||
4862 | /// Use3: | |||
4863 | /// reg(c) + reg(b) + reg({0,+,1}) | |||
4864 | /// reg(c) + reg({b,+,1}) | |||
4865 | /// | |||
4866 | /// Probability of not selecting | |||
4867 | /// Use1 Use2 Use3 | |||
4868 | /// reg(a) (1/3) * 1 * 1 | |||
4869 | /// reg(b) 1 * (1/3) * (1/2) | |||
4870 | /// reg({0,+,1}) (2/3) * (2/3) * (1/2) | |||
4871 | /// reg({-1,+,1}) (2/3) * (2/3) * 1 | |||
4872 | /// reg({a,+,1}) (2/3) * 1 * 1 | |||
4873 | /// reg({b,+,1}) 1 * (2/3) * (2/3) | |||
4874 | /// reg(c) 1 * 1 * 0 | |||
4875 | /// | |||
4876 | /// Now count registers number mathematical expectation for each formula: | |||
4877 | /// Note that for each use we exclude probability if not selecting for the use. | |||
4878 | /// For example for Use1 probability for reg(a) would be just 1 * 1 (excluding | |||
4879 | /// probabilty 1/3 of not selecting for Use1). | |||
4880 | /// Use1: | |||
4881 | /// reg(a) + reg({0,+,1}) 1 + 1/3 -- to be deleted | |||
4882 | /// reg(a) + reg({-1,+,1}) + 1 1 + 4/9 -- to be deleted | |||
4883 | /// reg({a,+,1}) 1 | |||
4884 | /// Use2: | |||
4885 | /// reg(b) + reg({0,+,1}) 1/2 + 1/3 -- to be deleted | |||
4886 | /// reg(b) + reg({-1,+,1}) + 1 1/2 + 2/3 -- to be deleted | |||
4887 | /// reg({b,+,1}) 2/3 | |||
4888 | /// Use3: | |||
4889 | /// reg(c) + reg(b) + reg({0,+,1}) 1 + 1/3 + 4/9 -- to be deleted | |||
4890 | /// reg(c) + reg({b,+,1}) 1 + 2/3 | |||
4891 | void LSRInstance::NarrowSearchSpaceByDeletingCostlyFormulas() { | |||
4892 | if (EstimateSearchSpaceComplexity() < ComplexityLimit) | |||
4893 | return; | |||
4894 | // Ok, we have too many of formulae on our hands to conveniently handle. | |||
4895 | // Use a rough heuristic to thin out the list. | |||
4896 | ||||
4897 | // Set of Regs wich will be 100% used in final solution. | |||
4898 | // Used in each formula of a solution (in example above this is reg(c)). | |||
4899 | // We can skip them in calculations. | |||
4900 | SmallPtrSet<const SCEV *, 4> UniqRegs; | |||
4901 | LLVM_DEBUG(dbgs() << "The search space is too complex.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" ; } } while (false); | |||
4902 | ||||
4903 | // Map each register to probability of not selecting | |||
4904 | DenseMap <const SCEV *, float> RegNumMap; | |||
4905 | for (const SCEV *Reg : RegUses) { | |||
4906 | if (UniqRegs.count(Reg)) | |||
4907 | continue; | |||
4908 | float PNotSel = 1; | |||
4909 | for (const LSRUse &LU : Uses) { | |||
4910 | if (!LU.Regs.count(Reg)) | |||
4911 | continue; | |||
4912 | float P = LU.getNotSelectedProbability(Reg); | |||
4913 | if (P != 0.0) | |||
4914 | PNotSel *= P; | |||
4915 | else | |||
4916 | UniqRegs.insert(Reg); | |||
4917 | } | |||
4918 | RegNumMap.insert(std::make_pair(Reg, PNotSel)); | |||
4919 | } | |||
4920 | ||||
4921 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Narrowing the search space by deleting costly formulas\n" ; } } while (false) | |||
4922 | dbgs() << "Narrowing the search space by deleting costly formulas\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Narrowing the search space by deleting costly formulas\n" ; } } while (false); | |||
4923 | ||||
4924 | // Delete formulas where registers number expectation is high. | |||
4925 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { | |||
4926 | LSRUse &LU = Uses[LUIdx]; | |||
4927 | // If nothing to delete - continue. | |||
4928 | if (LU.Formulae.size() < 2) | |||
4929 | continue; | |||
4930 | // This is temporary solution to test performance. Float should be | |||
4931 | // replaced with round independent type (based on integers) to avoid | |||
4932 | // different results for different target builds. | |||
4933 | float FMinRegNum = LU.Formulae[0].getNumRegs(); | |||
4934 | float FMinARegNum = LU.Formulae[0].getNumRegs(); | |||
4935 | size_t MinIdx = 0; | |||
4936 | for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { | |||
4937 | Formula &F = LU.Formulae[i]; | |||
4938 | float FRegNum = 0; | |||
4939 | float FARegNum = 0; | |||
4940 | for (const SCEV *BaseReg : F.BaseRegs) { | |||
4941 | if (UniqRegs.count(BaseReg)) | |||
4942 | continue; | |||
4943 | FRegNum += RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg); | |||
4944 | if (isa<SCEVAddRecExpr>(BaseReg)) | |||
4945 | FARegNum += | |||
4946 | RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg); | |||
4947 | } | |||
4948 | if (const SCEV *ScaledReg = F.ScaledReg) { | |||
4949 | if (!UniqRegs.count(ScaledReg)) { | |||
4950 | FRegNum += | |||
4951 | RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg); | |||
4952 | if (isa<SCEVAddRecExpr>(ScaledReg)) | |||
4953 | FARegNum += | |||
4954 | RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg); | |||
4955 | } | |||
4956 | } | |||
4957 | if (FMinRegNum > FRegNum || | |||
4958 | (FMinRegNum == FRegNum && FMinARegNum > FARegNum)) { | |||
4959 | FMinRegNum = FRegNum; | |||
4960 | FMinARegNum = FARegNum; | |||
4961 | MinIdx = i; | |||
4962 | } | |||
4963 | } | |||
4964 | LLVM_DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " The formula "; LU.Formulae [MinIdx].print(dbgs()); dbgs() << " with min reg num " << FMinRegNum << '\n'; } } while (false) | |||
4965 | dbgs() << " with min reg num " << FMinRegNum << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " The formula "; LU.Formulae [MinIdx].print(dbgs()); dbgs() << " with min reg num " << FMinRegNum << '\n'; } } while (false); | |||
4966 | if (MinIdx != 0) | |||
4967 | std::swap(LU.Formulae[MinIdx], LU.Formulae[0]); | |||
4968 | while (LU.Formulae.size() != 1) { | |||
4969 | LLVM_DEBUG(dbgs() << " Deleting "; LU.Formulae.back().print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Deleting "; LU.Formulae .back().print(dbgs()); dbgs() << '\n'; } } while (false ) | |||
4970 | dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Deleting "; LU.Formulae .back().print(dbgs()); dbgs() << '\n'; } } while (false ); | |||
4971 | LU.Formulae.pop_back(); | |||
4972 | } | |||
4973 | LU.RecomputeRegs(LUIdx, RegUses); | |||
4974 | assert(LU.Formulae.size() == 1 && "Should be exactly 1 min regs formula")(static_cast <bool> (LU.Formulae.size() == 1 && "Should be exactly 1 min regs formula") ? void (0) : __assert_fail ("LU.Formulae.size() == 1 && \"Should be exactly 1 min regs formula\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 4974, __extension__ __PRETTY_FUNCTION__)); | |||
4975 | Formula &F = LU.Formulae[0]; | |||
4976 | LLVM_DEBUG(dbgs() << " Leaving only "; F.print(dbgs()); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Leaving only "; F.print (dbgs()); dbgs() << '\n'; } } while (false); | |||
4977 | // When we choose the formula, the regs become unique. | |||
4978 | UniqRegs.insert(F.BaseRegs.begin(), F.BaseRegs.end()); | |||
4979 | if (F.ScaledReg) | |||
4980 | UniqRegs.insert(F.ScaledReg); | |||
4981 | } | |||
4982 | LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "After pre-selection:\n"; print_uses (dbgs()); } } while (false); | |||
4983 | } | |||
4984 | ||||
4985 | /// Pick a register which seems likely to be profitable, and then in any use | |||
4986 | /// which has any reference to that register, delete all formulae which do not | |||
4987 | /// reference that register. | |||
4988 | void LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() { | |||
4989 | // With all other options exhausted, loop until the system is simple | |||
4990 | // enough to handle. | |||
4991 | SmallPtrSet<const SCEV *, 4> Taken; | |||
4992 | while (EstimateSearchSpaceComplexity() >= ComplexityLimit) { | |||
4993 | // Ok, we have too many of formulae on our hands to conveniently handle. | |||
4994 | // Use a rough heuristic to thin out the list. | |||
4995 | LLVM_DEBUG(dbgs() << "The search space is too complex.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The search space is too complex.\n" ; } } while (false); | |||
4996 | ||||
4997 | // Pick the register which is used by the most LSRUses, which is likely | |||
4998 | // to be a good reuse register candidate. | |||
4999 | const SCEV *Best = nullptr; | |||
5000 | unsigned BestNum = 0; | |||
5001 | for (const SCEV *Reg : RegUses) { | |||
5002 | if (Taken.count(Reg)) | |||
5003 | continue; | |||
5004 | if (!Best) { | |||
5005 | Best = Reg; | |||
5006 | BestNum = RegUses.getUsedByIndices(Reg).count(); | |||
5007 | } else { | |||
5008 | unsigned Count = RegUses.getUsedByIndices(Reg).count(); | |||
5009 | if (Count > BestNum) { | |||
5010 | Best = Reg; | |||
5011 | BestNum = Count; | |||
5012 | } | |||
5013 | } | |||
5014 | } | |||
5015 | assert(Best && "Failed to find best LSRUse candidate")(static_cast <bool> (Best && "Failed to find best LSRUse candidate" ) ? void (0) : __assert_fail ("Best && \"Failed to find best LSRUse candidate\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5015, __extension__ __PRETTY_FUNCTION__)); | |||
5016 | ||||
5017 | LLVM_DEBUG(dbgs() << "Narrowing the search space by assuming " << *Bestdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Narrowing the search space by assuming " << *Best << " will yield profitable reuse.\n"; } } while (false) | |||
5018 | << " will yield profitable reuse.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Narrowing the search space by assuming " << *Best << " will yield profitable reuse.\n"; } } while (false); | |||
5019 | Taken.insert(Best); | |||
5020 | ||||
5021 | // In any use with formulae which references this register, delete formulae | |||
5022 | // which don't reference it. | |||
5023 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) { | |||
5024 | LSRUse &LU = Uses[LUIdx]; | |||
5025 | if (!LU.Regs.count(Best)) continue; | |||
5026 | ||||
5027 | bool Any = false; | |||
5028 | for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) { | |||
5029 | Formula &F = LU.Formulae[i]; | |||
5030 | if (!F.referencesReg(Best)) { | |||
5031 | LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << " Deleting "; F.print(dbgs ()); dbgs() << '\n'; } } while (false); | |||
5032 | LU.DeleteFormula(F); | |||
5033 | --e; | |||
5034 | --i; | |||
5035 | Any = true; | |||
5036 | assert(e != 0 && "Use has no formulae left! Is Regs inconsistent?")(static_cast <bool> (e != 0 && "Use has no formulae left! Is Regs inconsistent?" ) ? void (0) : __assert_fail ("e != 0 && \"Use has no formulae left! Is Regs inconsistent?\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5036, __extension__ __PRETTY_FUNCTION__)); | |||
5037 | continue; | |||
5038 | } | |||
5039 | } | |||
5040 | ||||
5041 | if (Any) | |||
5042 | LU.RecomputeRegs(LUIdx, RegUses); | |||
5043 | } | |||
5044 | ||||
5045 | LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "After pre-selection:\n"; print_uses (dbgs()); } } while (false); | |||
5046 | } | |||
5047 | } | |||
5048 | ||||
5049 | /// If there are an extraordinary number of formulae to choose from, use some | |||
5050 | /// rough heuristics to prune down the number of formulae. This keeps the main | |||
5051 | /// solver from taking an extraordinary amount of time in some worst-case | |||
5052 | /// scenarios. | |||
5053 | void LSRInstance::NarrowSearchSpaceUsingHeuristics() { | |||
5054 | NarrowSearchSpaceByDetectingSupersets(); | |||
5055 | NarrowSearchSpaceByCollapsingUnrolledCode(); | |||
5056 | NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(); | |||
5057 | if (FilterSameScaledReg) | |||
5058 | NarrowSearchSpaceByFilterFormulaWithSameScaledReg(); | |||
5059 | NarrowSearchSpaceByFilterPostInc(); | |||
5060 | if (LSRExpNarrow) | |||
5061 | NarrowSearchSpaceByDeletingCostlyFormulas(); | |||
5062 | else | |||
5063 | NarrowSearchSpaceByPickingWinnerRegs(); | |||
5064 | } | |||
5065 | ||||
5066 | /// This is the recursive solver. | |||
5067 | void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution, | |||
5068 | Cost &SolutionCost, | |||
5069 | SmallVectorImpl<const Formula *> &Workspace, | |||
5070 | const Cost &CurCost, | |||
5071 | const SmallPtrSet<const SCEV *, 16> &CurRegs, | |||
5072 | DenseSet<const SCEV *> &VisitedRegs) const { | |||
5073 | // Some ideas: | |||
5074 | // - prune more: | |||
5075 | // - use more aggressive filtering | |||
5076 | // - sort the formula so that the most profitable solutions are found first | |||
5077 | // - sort the uses too | |||
5078 | // - search faster: | |||
5079 | // - don't compute a cost, and then compare. compare while computing a cost | |||
5080 | // and bail early. | |||
5081 | // - track register sets with SmallBitVector | |||
5082 | ||||
5083 | const LSRUse &LU = Uses[Workspace.size()]; | |||
5084 | ||||
5085 | // If this use references any register that's already a part of the | |||
5086 | // in-progress solution, consider it a requirement that a formula must | |||
5087 | // reference that register in order to be considered. This prunes out | |||
5088 | // unprofitable searching. | |||
5089 | SmallSetVector<const SCEV *, 4> ReqRegs; | |||
5090 | for (const SCEV *S : CurRegs) | |||
5091 | if (LU.Regs.count(S)) | |||
5092 | ReqRegs.insert(S); | |||
5093 | ||||
5094 | SmallPtrSet<const SCEV *, 16> NewRegs; | |||
5095 | Cost NewCost(L, SE, TTI, AMK); | |||
5096 | for (const Formula &F : LU.Formulae) { | |||
5097 | // Ignore formulae which may not be ideal in terms of register reuse of | |||
5098 | // ReqRegs. The formula should use all required registers before | |||
5099 | // introducing new ones. | |||
5100 | // This can sometimes (notably when trying to favour postinc) lead to | |||
5101 | // sub-optimial decisions. There it is best left to the cost modelling to | |||
5102 | // get correct. | |||
5103 | if (AMK != TTI::AMK_PostIndexed || LU.Kind != LSRUse::Address) { | |||
5104 | int NumReqRegsToFind = std::min(F.getNumRegs(), ReqRegs.size()); | |||
5105 | for (const SCEV *Reg : ReqRegs) { | |||
5106 | if ((F.ScaledReg && F.ScaledReg == Reg) || | |||
5107 | is_contained(F.BaseRegs, Reg)) { | |||
5108 | --NumReqRegsToFind; | |||
5109 | if (NumReqRegsToFind == 0) | |||
5110 | break; | |||
5111 | } | |||
5112 | } | |||
5113 | if (NumReqRegsToFind != 0) { | |||
5114 | // If none of the formulae satisfied the required registers, then we could | |||
5115 | // clear ReqRegs and try again. Currently, we simply give up in this case. | |||
5116 | continue; | |||
5117 | } | |||
5118 | } | |||
5119 | ||||
5120 | // Evaluate the cost of the current formula. If it's already worse than | |||
5121 | // the current best, prune the search at that point. | |||
5122 | NewCost = CurCost; | |||
5123 | NewRegs = CurRegs; | |||
5124 | NewCost.RateFormula(F, NewRegs, VisitedRegs, LU); | |||
5125 | if (NewCost.isLess(SolutionCost)) { | |||
5126 | Workspace.push_back(&F); | |||
5127 | if (Workspace.size() != Uses.size()) { | |||
5128 | SolveRecurse(Solution, SolutionCost, Workspace, NewCost, | |||
5129 | NewRegs, VisitedRegs); | |||
5130 | if (F.getNumRegs() == 1 && Workspace.size() == 1) | |||
5131 | VisitedRegs.insert(F.ScaledReg ? F.ScaledReg : F.BaseRegs[0]); | |||
5132 | } else { | |||
5133 | LLVM_DEBUG(dbgs() << "New best at "; NewCost.print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "New best at "; NewCost.print (dbgs()); dbgs() << ".\nRegs:\n"; for (const SCEV *S : NewRegs ) dbgs() << "- " << *S << "\n"; dbgs() << '\n'; } } while (false) | |||
5134 | dbgs() << ".\nRegs:\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "New best at "; NewCost.print (dbgs()); dbgs() << ".\nRegs:\n"; for (const SCEV *S : NewRegs ) dbgs() << "- " << *S << "\n"; dbgs() << '\n'; } } while (false) | |||
5135 | for (const SCEV *S : NewRegs) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "New best at "; NewCost.print (dbgs()); dbgs() << ".\nRegs:\n"; for (const SCEV *S : NewRegs ) dbgs() << "- " << *S << "\n"; dbgs() << '\n'; } } while (false) | |||
5136 | << "- " << *S << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "New best at "; NewCost.print (dbgs()); dbgs() << ".\nRegs:\n"; for (const SCEV *S : NewRegs ) dbgs() << "- " << *S << "\n"; dbgs() << '\n'; } } while (false) | |||
5137 | dbgs() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "New best at "; NewCost.print (dbgs()); dbgs() << ".\nRegs:\n"; for (const SCEV *S : NewRegs ) dbgs() << "- " << *S << "\n"; dbgs() << '\n'; } } while (false); | |||
5138 | ||||
5139 | SolutionCost = NewCost; | |||
5140 | Solution = Workspace; | |||
5141 | } | |||
5142 | Workspace.pop_back(); | |||
5143 | } | |||
5144 | } | |||
5145 | } | |||
5146 | ||||
5147 | /// Choose one formula from each use. Return the results in the given Solution | |||
5148 | /// vector. | |||
5149 | void LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const { | |||
5150 | SmallVector<const Formula *, 8> Workspace; | |||
5151 | Cost SolutionCost(L, SE, TTI, AMK); | |||
5152 | SolutionCost.Lose(); | |||
5153 | Cost CurCost(L, SE, TTI, AMK); | |||
5154 | SmallPtrSet<const SCEV *, 16> CurRegs; | |||
5155 | DenseSet<const SCEV *> VisitedRegs; | |||
5156 | Workspace.reserve(Uses.size()); | |||
5157 | ||||
5158 | // SolveRecurse does all the work. | |||
5159 | SolveRecurse(Solution, SolutionCost, Workspace, CurCost, | |||
5160 | CurRegs, VisitedRegs); | |||
5161 | if (Solution.empty()) { | |||
5162 | LLVM_DEBUG(dbgs() << "\nNo Satisfactory Solution\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\nNo Satisfactory Solution\n" ; } } while (false); | |||
5163 | return; | |||
5164 | } | |||
5165 | ||||
5166 | // Ok, we've now made all our decisions. | |||
5167 | LLVM_DEBUG(dbgs() << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "The chosen solution requires " ; SolutionCost.print(dbgs()); dbgs() << ":\n"; for (size_t i = 0, e = Uses.size(); i != e; ++i) { dbgs() << " "; Uses[i].print(dbgs()); dbgs() << "\n" " "; Solution [i]->print(dbgs()); dbgs() << '\n'; }; } } while (false ) | |||
5168 | "The chosen solution requires ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "The chosen solution requires " ; SolutionCost.print(dbgs()); dbgs() << ":\n"; for (size_t i = 0, e = Uses.size(); i != e; ++i) { dbgs() << " "; Uses[i].print(dbgs()); dbgs() << "\n" " "; Solution [i]->print(dbgs()); dbgs() << '\n'; }; } } while (false ) | |||
5169 | SolutionCost.print(dbgs()); dbgs() << ":\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "The chosen solution requires " ; SolutionCost.print(dbgs()); dbgs() << ":\n"; for (size_t i = 0, e = Uses.size(); i != e; ++i) { dbgs() << " "; Uses[i].print(dbgs()); dbgs() << "\n" " "; Solution [i]->print(dbgs()); dbgs() << '\n'; }; } } while (false ) | |||
5170 | for (size_t i = 0, e = Uses.size(); i != e; ++i) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "The chosen solution requires " ; SolutionCost.print(dbgs()); dbgs() << ":\n"; for (size_t i = 0, e = Uses.size(); i != e; ++i) { dbgs() << " "; Uses[i].print(dbgs()); dbgs() << "\n" " "; Solution [i]->print(dbgs()); dbgs() << '\n'; }; } } while (false ) | |||
5171 | dbgs() << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "The chosen solution requires " ; SolutionCost.print(dbgs()); dbgs() << ":\n"; for (size_t i = 0, e = Uses.size(); i != e; ++i) { dbgs() << " "; Uses[i].print(dbgs()); dbgs() << "\n" " "; Solution [i]->print(dbgs()); dbgs() << '\n'; }; } } while (false ) | |||
5172 | Uses[i].print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "The chosen solution requires " ; SolutionCost.print(dbgs()); dbgs() << ":\n"; for (size_t i = 0, e = Uses.size(); i != e; ++i) { dbgs() << " "; Uses[i].print(dbgs()); dbgs() << "\n" " "; Solution [i]->print(dbgs()); dbgs() << '\n'; }; } } while (false ) | |||
5173 | dbgs() << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "The chosen solution requires " ; SolutionCost.print(dbgs()); dbgs() << ":\n"; for (size_t i = 0, e = Uses.size(); i != e; ++i) { dbgs() << " "; Uses[i].print(dbgs()); dbgs() << "\n" " "; Solution [i]->print(dbgs()); dbgs() << '\n'; }; } } while (false ) | |||
5174 | " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "The chosen solution requires " ; SolutionCost.print(dbgs()); dbgs() << ":\n"; for (size_t i = 0, e = Uses.size(); i != e; ++i) { dbgs() << " "; Uses[i].print(dbgs()); dbgs() << "\n" " "; Solution [i]->print(dbgs()); dbgs() << '\n'; }; } } while (false ) | |||
5175 | Solution[i]->print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "The chosen solution requires " ; SolutionCost.print(dbgs()); dbgs() << ":\n"; for (size_t i = 0, e = Uses.size(); i != e; ++i) { dbgs() << " "; Uses[i].print(dbgs()); dbgs() << "\n" " "; Solution [i]->print(dbgs()); dbgs() << '\n'; }; } } while (false ) | |||
5176 | dbgs() << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "The chosen solution requires " ; SolutionCost.print(dbgs()); dbgs() << ":\n"; for (size_t i = 0, e = Uses.size(); i != e; ++i) { dbgs() << " "; Uses[i].print(dbgs()); dbgs() << "\n" " "; Solution [i]->print(dbgs()); dbgs() << '\n'; }; } } while (false ) | |||
5177 | })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\n" "The chosen solution requires " ; SolutionCost.print(dbgs()); dbgs() << ":\n"; for (size_t i = 0, e = Uses.size(); i != e; ++i) { dbgs() << " "; Uses[i].print(dbgs()); dbgs() << "\n" " "; Solution [i]->print(dbgs()); dbgs() << '\n'; }; } } while (false ); | |||
5178 | ||||
5179 | assert(Solution.size() == Uses.size() && "Malformed solution!")(static_cast <bool> (Solution.size() == Uses.size() && "Malformed solution!") ? void (0) : __assert_fail ("Solution.size() == Uses.size() && \"Malformed solution!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5179, __extension__ __PRETTY_FUNCTION__)); | |||
5180 | ||||
5181 | if (BaselineCost.isLess(SolutionCost)) { | |||
5182 | LLVM_DEBUG(dbgs() << "The baseline solution requires ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The baseline solution requires " ; BaselineCost.print(dbgs()); dbgs() << "\n"; } } while (false) | |||
5183 | BaselineCost.print(dbgs()); dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "The baseline solution requires " ; BaselineCost.print(dbgs()); dbgs() << "\n"; } } while (false); | |||
5184 | if (!AllowDropSolutionIfLessProfitable) | |||
5185 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Baseline is more profitable than chosen solution, " "add option 'lsr-drop-solution' to drop LSR solution.\n"; } } while (false) | |||
5186 | dbgs() << "Baseline is more profitable than chosen solution, "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Baseline is more profitable than chosen solution, " "add option 'lsr-drop-solution' to drop LSR solution.\n"; } } while (false) | |||
5187 | "add option 'lsr-drop-solution' to drop LSR solution.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Baseline is more profitable than chosen solution, " "add option 'lsr-drop-solution' to drop LSR solution.\n"; } } while (false); | |||
5188 | else { | |||
5189 | LLVM_DEBUG(dbgs() << "Baseline is more profitable than chosen "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Baseline is more profitable than chosen " "solution, dropping LSR solution.\n";; } } while (false) | |||
5190 | "solution, dropping LSR solution.\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Baseline is more profitable than chosen " "solution, dropping LSR solution.\n";; } } while (false); | |||
5191 | Solution.clear(); | |||
5192 | } | |||
5193 | } | |||
5194 | } | |||
5195 | ||||
5196 | /// Helper for AdjustInsertPositionForExpand. Climb up the dominator tree far as | |||
5197 | /// we can go while still being dominated by the input positions. This helps | |||
5198 | /// canonicalize the insert position, which encourages sharing. | |||
5199 | BasicBlock::iterator | |||
5200 | LSRInstance::HoistInsertPosition(BasicBlock::iterator IP, | |||
5201 | const SmallVectorImpl<Instruction *> &Inputs) | |||
5202 | const { | |||
5203 | Instruction *Tentative = &*IP; | |||
5204 | while (true) { | |||
5205 | bool AllDominate = true; | |||
5206 | Instruction *BetterPos = nullptr; | |||
5207 | // Don't bother attempting to insert before a catchswitch, their basic block | |||
5208 | // cannot have other non-PHI instructions. | |||
5209 | if (isa<CatchSwitchInst>(Tentative)) | |||
5210 | return IP; | |||
5211 | ||||
5212 | for (Instruction *Inst : Inputs) { | |||
5213 | if (Inst == Tentative || !DT.dominates(Inst, Tentative)) { | |||
5214 | AllDominate = false; | |||
5215 | break; | |||
5216 | } | |||
5217 | // Attempt to find an insert position in the middle of the block, | |||
5218 | // instead of at the end, so that it can be used for other expansions. | |||
5219 | if (Tentative->getParent() == Inst->getParent() && | |||
5220 | (!BetterPos || !DT.dominates(Inst, BetterPos))) | |||
5221 | BetterPos = &*std::next(BasicBlock::iterator(Inst)); | |||
5222 | } | |||
5223 | if (!AllDominate) | |||
5224 | break; | |||
5225 | if (BetterPos) | |||
5226 | IP = BetterPos->getIterator(); | |||
5227 | else | |||
5228 | IP = Tentative->getIterator(); | |||
5229 | ||||
5230 | const Loop *IPLoop = LI.getLoopFor(IP->getParent()); | |||
5231 | unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : 0; | |||
5232 | ||||
5233 | BasicBlock *IDom; | |||
5234 | for (DomTreeNode *Rung = DT.getNode(IP->getParent()); ; ) { | |||
5235 | if (!Rung) return IP; | |||
5236 | Rung = Rung->getIDom(); | |||
5237 | if (!Rung) return IP; | |||
5238 | IDom = Rung->getBlock(); | |||
5239 | ||||
5240 | // Don't climb into a loop though. | |||
5241 | const Loop *IDomLoop = LI.getLoopFor(IDom); | |||
5242 | unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : 0; | |||
5243 | if (IDomDepth <= IPLoopDepth && | |||
5244 | (IDomDepth != IPLoopDepth || IDomLoop == IPLoop)) | |||
5245 | break; | |||
5246 | } | |||
5247 | ||||
5248 | Tentative = IDom->getTerminator(); | |||
5249 | } | |||
5250 | ||||
5251 | return IP; | |||
5252 | } | |||
5253 | ||||
5254 | /// Determine an input position which will be dominated by the operands and | |||
5255 | /// which will dominate the result. | |||
5256 | BasicBlock::iterator LSRInstance::AdjustInsertPositionForExpand( | |||
5257 | BasicBlock::iterator LowestIP, const LSRFixup &LF, const LSRUse &LU) const { | |||
5258 | // Collect some instructions which must be dominated by the | |||
5259 | // expanding replacement. These must be dominated by any operands that | |||
5260 | // will be required in the expansion. | |||
5261 | SmallVector<Instruction *, 4> Inputs; | |||
5262 | if (Instruction *I = dyn_cast<Instruction>(LF.OperandValToReplace)) | |||
5263 | Inputs.push_back(I); | |||
5264 | if (LU.Kind == LSRUse::ICmpZero) | |||
5265 | if (Instruction *I = | |||
5266 | dyn_cast<Instruction>(cast<ICmpInst>(LF.UserInst)->getOperand(1))) | |||
5267 | Inputs.push_back(I); | |||
5268 | if (LF.PostIncLoops.count(L)) { | |||
5269 | if (LF.isUseFullyOutsideLoop(L)) | |||
5270 | Inputs.push_back(L->getLoopLatch()->getTerminator()); | |||
5271 | else | |||
5272 | Inputs.push_back(IVIncInsertPos); | |||
5273 | } | |||
5274 | // The expansion must also be dominated by the increment positions of any | |||
5275 | // loops it for which it is using post-inc mode. | |||
5276 | for (const Loop *PIL : LF.PostIncLoops) { | |||
5277 | if (PIL == L) continue; | |||
5278 | ||||
5279 | // Be dominated by the loop exit. | |||
5280 | SmallVector<BasicBlock *, 4> ExitingBlocks; | |||
5281 | PIL->getExitingBlocks(ExitingBlocks); | |||
5282 | if (!ExitingBlocks.empty()) { | |||
5283 | BasicBlock *BB = ExitingBlocks[0]; | |||
5284 | for (unsigned i = 1, e = ExitingBlocks.size(); i != e; ++i) | |||
5285 | BB = DT.findNearestCommonDominator(BB, ExitingBlocks[i]); | |||
5286 | Inputs.push_back(BB->getTerminator()); | |||
5287 | } | |||
5288 | } | |||
5289 | ||||
5290 | assert(!isa<PHINode>(LowestIP) && !LowestIP->isEHPad()(static_cast <bool> (!isa<PHINode>(LowestIP) && !LowestIP->isEHPad() && !isa<DbgInfoIntrinsic> (LowestIP) && "Insertion point must be a normal instruction" ) ? void (0) : __assert_fail ("!isa<PHINode>(LowestIP) && !LowestIP->isEHPad() && !isa<DbgInfoIntrinsic>(LowestIP) && \"Insertion point must be a normal instruction\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5292, __extension__ __PRETTY_FUNCTION__)) | |||
5291 | && !isa<DbgInfoIntrinsic>(LowestIP) &&(static_cast <bool> (!isa<PHINode>(LowestIP) && !LowestIP->isEHPad() && !isa<DbgInfoIntrinsic> (LowestIP) && "Insertion point must be a normal instruction" ) ? void (0) : __assert_fail ("!isa<PHINode>(LowestIP) && !LowestIP->isEHPad() && !isa<DbgInfoIntrinsic>(LowestIP) && \"Insertion point must be a normal instruction\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5292, __extension__ __PRETTY_FUNCTION__)) | |||
5292 | "Insertion point must be a normal instruction")(static_cast <bool> (!isa<PHINode>(LowestIP) && !LowestIP->isEHPad() && !isa<DbgInfoIntrinsic> (LowestIP) && "Insertion point must be a normal instruction" ) ? void (0) : __assert_fail ("!isa<PHINode>(LowestIP) && !LowestIP->isEHPad() && !isa<DbgInfoIntrinsic>(LowestIP) && \"Insertion point must be a normal instruction\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5292, __extension__ __PRETTY_FUNCTION__)); | |||
5293 | ||||
5294 | // Then, climb up the immediate dominator tree as far as we can go while | |||
5295 | // still being dominated by the input positions. | |||
5296 | BasicBlock::iterator IP = HoistInsertPosition(LowestIP, Inputs); | |||
5297 | ||||
5298 | // Don't insert instructions before PHI nodes. | |||
5299 | while (isa<PHINode>(IP)) ++IP; | |||
5300 | ||||
5301 | // Ignore landingpad instructions. | |||
5302 | while (IP->isEHPad()) ++IP; | |||
5303 | ||||
5304 | // Ignore debug intrinsics. | |||
5305 | while (isa<DbgInfoIntrinsic>(IP)) ++IP; | |||
5306 | ||||
5307 | // Set IP below instructions recently inserted by SCEVExpander. This keeps the | |||
5308 | // IP consistent across expansions and allows the previously inserted | |||
5309 | // instructions to be reused by subsequent expansion. | |||
5310 | while (Rewriter.isInsertedInstruction(&*IP) && IP != LowestIP) | |||
5311 | ++IP; | |||
5312 | ||||
5313 | return IP; | |||
5314 | } | |||
5315 | ||||
5316 | /// Emit instructions for the leading candidate expression for this LSRUse (this | |||
5317 | /// is called "expanding"). | |||
5318 | Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF, | |||
5319 | const Formula &F, BasicBlock::iterator IP, | |||
5320 | SmallVectorImpl<WeakTrackingVH> &DeadInsts) const { | |||
5321 | if (LU.RigidFormula) | |||
5322 | return LF.OperandValToReplace; | |||
5323 | ||||
5324 | // Determine an input position which will be dominated by the operands and | |||
5325 | // which will dominate the result. | |||
5326 | IP = AdjustInsertPositionForExpand(IP, LF, LU); | |||
5327 | Rewriter.setInsertPoint(&*IP); | |||
5328 | ||||
5329 | // Inform the Rewriter if we have a post-increment use, so that it can | |||
5330 | // perform an advantageous expansion. | |||
5331 | Rewriter.setPostInc(LF.PostIncLoops); | |||
5332 | ||||
5333 | // This is the type that the user actually needs. | |||
5334 | Type *OpTy = LF.OperandValToReplace->getType(); | |||
5335 | // This will be the type that we'll initially expand to. | |||
5336 | Type *Ty = F.getType(); | |||
5337 | if (!Ty) | |||
5338 | // No type known; just expand directly to the ultimate type. | |||
5339 | Ty = OpTy; | |||
5340 | else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(OpTy)) | |||
5341 | // Expand directly to the ultimate type if it's the right size. | |||
5342 | Ty = OpTy; | |||
5343 | // This is the type to do integer arithmetic in. | |||
5344 | Type *IntTy = SE.getEffectiveSCEVType(Ty); | |||
5345 | ||||
5346 | // Build up a list of operands to add together to form the full base. | |||
5347 | SmallVector<const SCEV *, 8> Ops; | |||
5348 | ||||
5349 | // Expand the BaseRegs portion. | |||
5350 | for (const SCEV *Reg : F.BaseRegs) { | |||
5351 | assert(!Reg->isZero() && "Zero allocated in a base register!")(static_cast <bool> (!Reg->isZero() && "Zero allocated in a base register!" ) ? void (0) : __assert_fail ("!Reg->isZero() && \"Zero allocated in a base register!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5351, __extension__ __PRETTY_FUNCTION__)); | |||
5352 | ||||
5353 | // If we're expanding for a post-inc user, make the post-inc adjustment. | |||
5354 | Reg = denormalizeForPostIncUse(Reg, LF.PostIncLoops, SE); | |||
5355 | Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, nullptr))); | |||
5356 | } | |||
5357 | ||||
5358 | // Expand the ScaledReg portion. | |||
5359 | Value *ICmpScaledV = nullptr; | |||
5360 | if (F.Scale != 0) { | |||
5361 | const SCEV *ScaledS = F.ScaledReg; | |||
5362 | ||||
5363 | // If we're expanding for a post-inc user, make the post-inc adjustment. | |||
5364 | PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops); | |||
5365 | ScaledS = denormalizeForPostIncUse(ScaledS, Loops, SE); | |||
5366 | ||||
5367 | if (LU.Kind == LSRUse::ICmpZero) { | |||
5368 | // Expand ScaleReg as if it was part of the base regs. | |||
5369 | if (F.Scale == 1) | |||
5370 | Ops.push_back( | |||
5371 | SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr))); | |||
5372 | else { | |||
5373 | // An interesting way of "folding" with an icmp is to use a negated | |||
5374 | // scale, which we'll implement by inserting it into the other operand | |||
5375 | // of the icmp. | |||
5376 | assert(F.Scale == -1 &&(static_cast <bool> (F.Scale == -1 && "The only scale supported by ICmpZero uses is -1!" ) ? void (0) : __assert_fail ("F.Scale == -1 && \"The only scale supported by ICmpZero uses is -1!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5377, __extension__ __PRETTY_FUNCTION__)) | |||
5377 | "The only scale supported by ICmpZero uses is -1!")(static_cast <bool> (F.Scale == -1 && "The only scale supported by ICmpZero uses is -1!" ) ? void (0) : __assert_fail ("F.Scale == -1 && \"The only scale supported by ICmpZero uses is -1!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5377, __extension__ __PRETTY_FUNCTION__)); | |||
5378 | ICmpScaledV = Rewriter.expandCodeFor(ScaledS, nullptr); | |||
5379 | } | |||
5380 | } else { | |||
5381 | // Otherwise just expand the scaled register and an explicit scale, | |||
5382 | // which is expected to be matched as part of the address. | |||
5383 | ||||
5384 | // Flush the operand list to suppress SCEVExpander hoisting address modes. | |||
5385 | // Unless the addressing mode will not be folded. | |||
5386 | if (!Ops.empty() && LU.Kind == LSRUse::Address && | |||
5387 | isAMCompletelyFolded(TTI, LU, F)) { | |||
5388 | Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), nullptr); | |||
5389 | Ops.clear(); | |||
5390 | Ops.push_back(SE.getUnknown(FullV)); | |||
5391 | } | |||
5392 | ScaledS = SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr)); | |||
5393 | if (F.Scale != 1) | |||
5394 | ScaledS = | |||
5395 | SE.getMulExpr(ScaledS, SE.getConstant(ScaledS->getType(), F.Scale)); | |||
5396 | Ops.push_back(ScaledS); | |||
5397 | } | |||
5398 | } | |||
5399 | ||||
5400 | // Expand the GV portion. | |||
5401 | if (F.BaseGV) { | |||
5402 | // Flush the operand list to suppress SCEVExpander hoisting. | |||
5403 | if (!Ops.empty()) { | |||
5404 | Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), IntTy); | |||
5405 | Ops.clear(); | |||
5406 | Ops.push_back(SE.getUnknown(FullV)); | |||
5407 | } | |||
5408 | Ops.push_back(SE.getUnknown(F.BaseGV)); | |||
5409 | } | |||
5410 | ||||
5411 | // Flush the operand list to suppress SCEVExpander hoisting of both folded and | |||
5412 | // unfolded offsets. LSR assumes they both live next to their uses. | |||
5413 | if (!Ops.empty()) { | |||
5414 | Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty); | |||
5415 | Ops.clear(); | |||
5416 | Ops.push_back(SE.getUnknown(FullV)); | |||
5417 | } | |||
5418 | ||||
5419 | // Expand the immediate portion. | |||
5420 | int64_t Offset = (uint64_t)F.BaseOffset + LF.Offset; | |||
5421 | if (Offset != 0) { | |||
5422 | if (LU.Kind == LSRUse::ICmpZero) { | |||
5423 | // The other interesting way of "folding" with an ICmpZero is to use a | |||
5424 | // negated immediate. | |||
5425 | if (!ICmpScaledV) | |||
5426 | ICmpScaledV = ConstantInt::get(IntTy, -(uint64_t)Offset); | |||
5427 | else { | |||
5428 | Ops.push_back(SE.getUnknown(ICmpScaledV)); | |||
5429 | ICmpScaledV = ConstantInt::get(IntTy, Offset); | |||
5430 | } | |||
5431 | } else { | |||
5432 | // Just add the immediate values. These again are expected to be matched | |||
5433 | // as part of the address. | |||
5434 | Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, Offset))); | |||
5435 | } | |||
5436 | } | |||
5437 | ||||
5438 | // Expand the unfolded offset portion. | |||
5439 | int64_t UnfoldedOffset = F.UnfoldedOffset; | |||
5440 | if (UnfoldedOffset != 0) { | |||
5441 | // Just add the immediate values. | |||
5442 | Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, | |||
5443 | UnfoldedOffset))); | |||
5444 | } | |||
5445 | ||||
5446 | // Emit instructions summing all the operands. | |||
5447 | const SCEV *FullS = Ops.empty() ? | |||
5448 | SE.getConstant(IntTy, 0) : | |||
5449 | SE.getAddExpr(Ops); | |||
5450 | Value *FullV = Rewriter.expandCodeFor(FullS, Ty); | |||
5451 | ||||
5452 | // We're done expanding now, so reset the rewriter. | |||
5453 | Rewriter.clearPostInc(); | |||
5454 | ||||
5455 | // An ICmpZero Formula represents an ICmp which we're handling as a | |||
5456 | // comparison against zero. Now that we've expanded an expression for that | |||
5457 | // form, update the ICmp's other operand. | |||
5458 | if (LU.Kind == LSRUse::ICmpZero) { | |||
5459 | ICmpInst *CI = cast<ICmpInst>(LF.UserInst); | |||
5460 | if (auto *OperandIsInstr = dyn_cast<Instruction>(CI->getOperand(1))) | |||
5461 | DeadInsts.emplace_back(OperandIsInstr); | |||
5462 | assert(!F.BaseGV && "ICmp does not support folding a global value and "(static_cast <bool> (!F.BaseGV && "ICmp does not support folding a global value and " "a scale at the same time!") ? void (0) : __assert_fail ("!F.BaseGV && \"ICmp does not support folding a global value and \" \"a scale at the same time!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5463, __extension__ __PRETTY_FUNCTION__)) | |||
5463 | "a scale at the same time!")(static_cast <bool> (!F.BaseGV && "ICmp does not support folding a global value and " "a scale at the same time!") ? void (0) : __assert_fail ("!F.BaseGV && \"ICmp does not support folding a global value and \" \"a scale at the same time!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5463, __extension__ __PRETTY_FUNCTION__)); | |||
5464 | if (F.Scale == -1) { | |||
5465 | if (ICmpScaledV->getType() != OpTy) { | |||
5466 | Instruction *Cast = | |||
5467 | CastInst::Create(CastInst::getCastOpcode(ICmpScaledV, false, | |||
5468 | OpTy, false), | |||
5469 | ICmpScaledV, OpTy, "tmp", CI); | |||
5470 | ICmpScaledV = Cast; | |||
5471 | } | |||
5472 | CI->setOperand(1, ICmpScaledV); | |||
5473 | } else { | |||
5474 | // A scale of 1 means that the scale has been expanded as part of the | |||
5475 | // base regs. | |||
5476 | assert((F.Scale == 0 || F.Scale == 1) &&(static_cast <bool> ((F.Scale == 0 || F.Scale == 1) && "ICmp does not support folding a global value and " "a scale at the same time!" ) ? void (0) : __assert_fail ("(F.Scale == 0 || F.Scale == 1) && \"ICmp does not support folding a global value and \" \"a scale at the same time!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5478, __extension__ __PRETTY_FUNCTION__)) | |||
5477 | "ICmp does not support folding a global value and "(static_cast <bool> ((F.Scale == 0 || F.Scale == 1) && "ICmp does not support folding a global value and " "a scale at the same time!" ) ? void (0) : __assert_fail ("(F.Scale == 0 || F.Scale == 1) && \"ICmp does not support folding a global value and \" \"a scale at the same time!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5478, __extension__ __PRETTY_FUNCTION__)) | |||
5478 | "a scale at the same time!")(static_cast <bool> ((F.Scale == 0 || F.Scale == 1) && "ICmp does not support folding a global value and " "a scale at the same time!" ) ? void (0) : __assert_fail ("(F.Scale == 0 || F.Scale == 1) && \"ICmp does not support folding a global value and \" \"a scale at the same time!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5478, __extension__ __PRETTY_FUNCTION__)); | |||
5479 | Constant *C = ConstantInt::getSigned(SE.getEffectiveSCEVType(OpTy), | |||
5480 | -(uint64_t)Offset); | |||
5481 | if (C->getType() != OpTy) | |||
5482 | C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false, | |||
5483 | OpTy, false), | |||
5484 | C, OpTy); | |||
5485 | ||||
5486 | CI->setOperand(1, C); | |||
5487 | } | |||
5488 | } | |||
5489 | ||||
5490 | return FullV; | |||
5491 | } | |||
5492 | ||||
5493 | /// Helper for Rewrite. PHI nodes are special because the use of their operands | |||
5494 | /// effectively happens in their predecessor blocks, so the expression may need | |||
5495 | /// to be expanded in multiple places. | |||
5496 | void LSRInstance::RewriteForPHI( | |||
5497 | PHINode *PN, const LSRUse &LU, const LSRFixup &LF, const Formula &F, | |||
5498 | SmallVectorImpl<WeakTrackingVH> &DeadInsts) const { | |||
5499 | DenseMap<BasicBlock *, Value *> Inserted; | |||
5500 | for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) | |||
5501 | if (PN->getIncomingValue(i) == LF.OperandValToReplace) { | |||
5502 | bool needUpdateFixups = false; | |||
5503 | BasicBlock *BB = PN->getIncomingBlock(i); | |||
5504 | ||||
5505 | // If this is a critical edge, split the edge so that we do not insert | |||
5506 | // the code on all predecessor/successor paths. We do this unless this | |||
5507 | // is the canonical backedge for this loop, which complicates post-inc | |||
5508 | // users. | |||
5509 | if (e != 1 && BB->getTerminator()->getNumSuccessors() > 1 && | |||
5510 | !isa<IndirectBrInst>(BB->getTerminator()) && | |||
5511 | !isa<CatchSwitchInst>(BB->getTerminator())) { | |||
5512 | BasicBlock *Parent = PN->getParent(); | |||
5513 | Loop *PNLoop = LI.getLoopFor(Parent); | |||
5514 | if (!PNLoop || Parent != PNLoop->getHeader()) { | |||
5515 | // Split the critical edge. | |||
5516 | BasicBlock *NewBB = nullptr; | |||
5517 | if (!Parent->isLandingPad()) { | |||
5518 | NewBB = | |||
5519 | SplitCriticalEdge(BB, Parent, | |||
5520 | CriticalEdgeSplittingOptions(&DT, &LI, MSSAU) | |||
5521 | .setMergeIdenticalEdges() | |||
5522 | .setKeepOneInputPHIs()); | |||
5523 | } else { | |||
5524 | SmallVector<BasicBlock*, 2> NewBBs; | |||
5525 | SplitLandingPadPredecessors(Parent, BB, "", "", NewBBs, &DT, &LI); | |||
5526 | NewBB = NewBBs[0]; | |||
5527 | } | |||
5528 | // If NewBB==NULL, then SplitCriticalEdge refused to split because all | |||
5529 | // phi predecessors are identical. The simple thing to do is skip | |||
5530 | // splitting in this case rather than complicate the API. | |||
5531 | if (NewBB) { | |||
5532 | // If PN is outside of the loop and BB is in the loop, we want to | |||
5533 | // move the block to be immediately before the PHI block, not | |||
5534 | // immediately after BB. | |||
5535 | if (L->contains(BB) && !L->contains(PN)) | |||
5536 | NewBB->moveBefore(PN->getParent()); | |||
5537 | ||||
5538 | // Splitting the edge can reduce the number of PHI entries we have. | |||
5539 | e = PN->getNumIncomingValues(); | |||
5540 | BB = NewBB; | |||
5541 | i = PN->getBasicBlockIndex(BB); | |||
5542 | ||||
5543 | needUpdateFixups = true; | |||
5544 | } | |||
5545 | } | |||
5546 | } | |||
5547 | ||||
5548 | std::pair<DenseMap<BasicBlock *, Value *>::iterator, bool> Pair = | |||
5549 | Inserted.insert(std::make_pair(BB, static_cast<Value *>(nullptr))); | |||
5550 | if (!Pair.second) | |||
5551 | PN->setIncomingValue(i, Pair.first->second); | |||
5552 | else { | |||
5553 | Value *FullV = | |||
5554 | Expand(LU, LF, F, BB->getTerminator()->getIterator(), DeadInsts); | |||
5555 | ||||
5556 | // If this is reuse-by-noop-cast, insert the noop cast. | |||
5557 | Type *OpTy = LF.OperandValToReplace->getType(); | |||
5558 | if (FullV->getType() != OpTy) | |||
5559 | FullV = | |||
5560 | CastInst::Create(CastInst::getCastOpcode(FullV, false, | |||
5561 | OpTy, false), | |||
5562 | FullV, LF.OperandValToReplace->getType(), | |||
5563 | "tmp", BB->getTerminator()); | |||
5564 | ||||
5565 | PN->setIncomingValue(i, FullV); | |||
5566 | Pair.first->second = FullV; | |||
5567 | } | |||
5568 | ||||
5569 | // If LSR splits critical edge and phi node has other pending | |||
5570 | // fixup operands, we need to update those pending fixups. Otherwise | |||
5571 | // formulae will not be implemented completely and some instructions | |||
5572 | // will not be eliminated. | |||
5573 | if (needUpdateFixups) { | |||
5574 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) | |||
5575 | for (LSRFixup &Fixup : Uses[LUIdx].Fixups) | |||
5576 | // If fixup is supposed to rewrite some operand in the phi | |||
5577 | // that was just updated, it may be already moved to | |||
5578 | // another phi node. Such fixup requires update. | |||
5579 | if (Fixup.UserInst == PN) { | |||
5580 | // Check if the operand we try to replace still exists in the | |||
5581 | // original phi. | |||
5582 | bool foundInOriginalPHI = false; | |||
5583 | for (const auto &val : PN->incoming_values()) | |||
5584 | if (val == Fixup.OperandValToReplace) { | |||
5585 | foundInOriginalPHI = true; | |||
5586 | break; | |||
5587 | } | |||
5588 | ||||
5589 | // If fixup operand found in original PHI - nothing to do. | |||
5590 | if (foundInOriginalPHI) | |||
5591 | continue; | |||
5592 | ||||
5593 | // Otherwise it might be moved to another PHI and requires update. | |||
5594 | // If fixup operand not found in any of the incoming blocks that | |||
5595 | // means we have already rewritten it - nothing to do. | |||
5596 | for (const auto &Block : PN->blocks()) | |||
5597 | for (BasicBlock::iterator I = Block->begin(); isa<PHINode>(I); | |||
5598 | ++I) { | |||
5599 | PHINode *NewPN = cast<PHINode>(I); | |||
5600 | for (const auto &val : NewPN->incoming_values()) | |||
5601 | if (val == Fixup.OperandValToReplace) | |||
5602 | Fixup.UserInst = NewPN; | |||
5603 | } | |||
5604 | } | |||
5605 | } | |||
5606 | } | |||
5607 | } | |||
5608 | ||||
5609 | /// Emit instructions for the leading candidate expression for this LSRUse (this | |||
5610 | /// is called "expanding"), and update the UserInst to reference the newly | |||
5611 | /// expanded value. | |||
5612 | void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF, | |||
5613 | const Formula &F, | |||
5614 | SmallVectorImpl<WeakTrackingVH> &DeadInsts) const { | |||
5615 | // First, find an insertion point that dominates UserInst. For PHI nodes, | |||
5616 | // find the nearest block which dominates all the relevant uses. | |||
5617 | if (PHINode *PN = dyn_cast<PHINode>(LF.UserInst)) { | |||
5618 | RewriteForPHI(PN, LU, LF, F, DeadInsts); | |||
5619 | } else { | |||
5620 | Value *FullV = Expand(LU, LF, F, LF.UserInst->getIterator(), DeadInsts); | |||
5621 | ||||
5622 | // If this is reuse-by-noop-cast, insert the noop cast. | |||
5623 | Type *OpTy = LF.OperandValToReplace->getType(); | |||
5624 | if (FullV->getType() != OpTy) { | |||
5625 | Instruction *Cast = | |||
5626 | CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false), | |||
5627 | FullV, OpTy, "tmp", LF.UserInst); | |||
5628 | FullV = Cast; | |||
5629 | } | |||
5630 | ||||
5631 | // Update the user. ICmpZero is handled specially here (for now) because | |||
5632 | // Expand may have updated one of the operands of the icmp already, and | |||
5633 | // its new value may happen to be equal to LF.OperandValToReplace, in | |||
5634 | // which case doing replaceUsesOfWith leads to replacing both operands | |||
5635 | // with the same value. TODO: Reorganize this. | |||
5636 | if (LU.Kind == LSRUse::ICmpZero) | |||
5637 | LF.UserInst->setOperand(0, FullV); | |||
5638 | else | |||
5639 | LF.UserInst->replaceUsesOfWith(LF.OperandValToReplace, FullV); | |||
5640 | } | |||
5641 | ||||
5642 | if (auto *OperandIsInstr = dyn_cast<Instruction>(LF.OperandValToReplace)) | |||
5643 | DeadInsts.emplace_back(OperandIsInstr); | |||
5644 | } | |||
5645 | ||||
5646 | /// Rewrite all the fixup locations with new values, following the chosen | |||
5647 | /// solution. | |||
5648 | void LSRInstance::ImplementSolution( | |||
5649 | const SmallVectorImpl<const Formula *> &Solution) { | |||
5650 | // Keep track of instructions we may have made dead, so that | |||
5651 | // we can remove them after we are done working. | |||
5652 | SmallVector<WeakTrackingVH, 16> DeadInsts; | |||
5653 | ||||
5654 | Rewriter.setIVIncInsertPos(L, IVIncInsertPos); | |||
5655 | ||||
5656 | // Mark phi nodes that terminate chains so the expander tries to reuse them. | |||
5657 | for (const IVChain &Chain : IVChainVec) { | |||
5658 | if (PHINode *PN = dyn_cast<PHINode>(Chain.tailUserInst())) | |||
5659 | Rewriter.setChainedPhi(PN); | |||
5660 | } | |||
5661 | ||||
5662 | // Expand the new value definitions and update the users. | |||
5663 | for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) | |||
5664 | for (const LSRFixup &Fixup : Uses[LUIdx].Fixups) { | |||
5665 | Rewrite(Uses[LUIdx], Fixup, *Solution[LUIdx], DeadInsts); | |||
5666 | Changed = true; | |||
5667 | } | |||
5668 | ||||
5669 | for (const IVChain &Chain : IVChainVec) { | |||
5670 | GenerateIVChain(Chain, DeadInsts); | |||
5671 | Changed = true; | |||
5672 | } | |||
5673 | ||||
5674 | for (const WeakVH &IV : Rewriter.getInsertedIVs()) | |||
5675 | if (IV && dyn_cast<Instruction>(&*IV)->getParent()) | |||
| ||||
5676 | ScalarEvolutionIVs.push_back(IV); | |||
5677 | ||||
5678 | // Clean up after ourselves. This must be done before deleting any | |||
5679 | // instructions. | |||
5680 | Rewriter.clear(); | |||
5681 | ||||
5682 | Changed |= RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts, | |||
5683 | &TLI, MSSAU); | |||
5684 | ||||
5685 | // In our cost analysis above, we assume that each addrec consumes exactly | |||
5686 | // one register, and arrange to have increments inserted just before the | |||
5687 | // latch to maximimize the chance this is true. However, if we reused | |||
5688 | // existing IVs, we now need to move the increments to match our | |||
5689 | // expectations. Otherwise, our cost modeling results in us having a | |||
5690 | // chosen a non-optimal result for the actual schedule. (And yes, this | |||
5691 | // scheduling decision does impact later codegen.) | |||
5692 | for (PHINode &PN : L->getHeader()->phis()) { | |||
5693 | BinaryOperator *BO = nullptr; | |||
5694 | Value *Start = nullptr, *Step = nullptr; | |||
5695 | if (!matchSimpleRecurrence(&PN, BO, Start, Step)) | |||
5696 | continue; | |||
5697 | ||||
5698 | switch (BO->getOpcode()) { | |||
5699 | case Instruction::Sub: | |||
5700 | if (BO->getOperand(0) != &PN) | |||
5701 | // sub is non-commutative - match handling elsewhere in LSR | |||
5702 | continue; | |||
5703 | break; | |||
5704 | case Instruction::Add: | |||
5705 | break; | |||
5706 | default: | |||
5707 | continue; | |||
5708 | }; | |||
5709 | ||||
5710 | if (!isa<Constant>(Step)) | |||
5711 | // If not a constant step, might increase register pressure | |||
5712 | // (We assume constants have been canonicalized to RHS) | |||
5713 | continue; | |||
5714 | ||||
5715 | if (BO->getParent() == IVIncInsertPos->getParent()) | |||
5716 | // Only bother moving across blocks. Isel can handle block local case. | |||
5717 | continue; | |||
5718 | ||||
5719 | // Can we legally schedule inc at the desired point? | |||
5720 | if (!llvm::all_of(BO->uses(), | |||
5721 | [&](Use &U) {return DT.dominates(IVIncInsertPos, U);})) | |||
5722 | continue; | |||
5723 | BO->moveBefore(IVIncInsertPos); | |||
5724 | Changed = true; | |||
5725 | } | |||
5726 | ||||
5727 | ||||
5728 | } | |||
5729 | ||||
5730 | LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE, | |||
5731 | DominatorTree &DT, LoopInfo &LI, | |||
5732 | const TargetTransformInfo &TTI, AssumptionCache &AC, | |||
5733 | TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU) | |||
5734 | : IU(IU), SE(SE), DT(DT), LI(LI), AC(AC), TLI(TLI), TTI(TTI), L(L), | |||
5735 | MSSAU(MSSAU), AMK(PreferredAddresingMode.getNumOccurrences() > 0 | |||
5736 | ? PreferredAddresingMode | |||
5737 | : TTI.getPreferredAddressingMode(L, &SE)), | |||
5738 | Rewriter(SE, L->getHeader()->getModule()->getDataLayout(), "lsr", false), | |||
5739 | BaselineCost(L, SE, TTI, AMK) { | |||
5740 | // If LoopSimplify form is not available, stay out of trouble. | |||
5741 | if (!L->isLoopSimplifyForm()) | |||
5742 | return; | |||
5743 | ||||
5744 | // If there's no interesting work to be done, bail early. | |||
5745 | if (IU.empty()) return; | |||
5746 | ||||
5747 | // If there's too much analysis to be done, bail early. We won't be able to | |||
5748 | // model the problem anyway. | |||
5749 | unsigned NumUsers = 0; | |||
5750 | for (const IVStrideUse &U : IU) { | |||
5751 | if (++NumUsers > MaxIVUsers) { | |||
5752 | (void)U; | |||
5753 | LLVM_DEBUG(dbgs() << "LSR skipping loop, too many IV Users in " << Udo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "LSR skipping loop, too many IV Users in " << U << "\n"; } } while (false) | |||
5754 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "LSR skipping loop, too many IV Users in " << U << "\n"; } } while (false); | |||
5755 | return; | |||
5756 | } | |||
5757 | // Bail out if we have a PHI on an EHPad that gets a value from a | |||
5758 | // CatchSwitchInst. Because the CatchSwitchInst cannot be split, there is | |||
5759 | // no good place to stick any instructions. | |||
5760 | if (auto *PN = dyn_cast<PHINode>(U.getUser())) { | |||
5761 | auto *FirstNonPHI = PN->getParent()->getFirstNonPHI(); | |||
5762 | if (isa<FuncletPadInst>(FirstNonPHI) || | |||
5763 | isa<CatchSwitchInst>(FirstNonPHI)) | |||
5764 | for (BasicBlock *PredBB : PN->blocks()) | |||
5765 | if (isa<CatchSwitchInst>(PredBB->getFirstNonPHI())) | |||
5766 | return; | |||
5767 | } | |||
5768 | } | |||
5769 | ||||
5770 | LLVM_DEBUG(dbgs() << "\nLSR on loop ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\nLSR on loop "; L->getHeader ()->printAsOperand(dbgs(), false); dbgs() << ":\n"; } } while (false) | |||
5771 | L->getHeader()->printAsOperand(dbgs(), /*PrintType=*/false);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\nLSR on loop "; L->getHeader ()->printAsOperand(dbgs(), false); dbgs() << ":\n"; } } while (false) | |||
5772 | dbgs() << ":\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "\nLSR on loop "; L->getHeader ()->printAsOperand(dbgs(), false); dbgs() << ":\n"; } } while (false); | |||
5773 | ||||
5774 | // Configure SCEVExpander already now, so the correct mode is used for | |||
5775 | // isSafeToExpand() checks. | |||
5776 | #ifndef NDEBUG | |||
5777 | Rewriter.setDebugType(DEBUG_TYPE"loop-reduce"); | |||
5778 | #endif | |||
5779 | Rewriter.disableCanonicalMode(); | |||
5780 | Rewriter.enableLSRMode(); | |||
5781 | ||||
5782 | // First, perform some low-level loop optimizations. | |||
5783 | OptimizeShadowIV(); | |||
5784 | OptimizeLoopTermCond(); | |||
5785 | ||||
5786 | // If loop preparation eliminates all interesting IV users, bail. | |||
5787 | if (IU.empty()) return; | |||
5788 | ||||
5789 | // Skip nested loops until we can model them better with formulae. | |||
5790 | if (!L->isInnermost()) { | |||
5791 | LLVM_DEBUG(dbgs() << "LSR skipping outer loop " << *L << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "LSR skipping outer loop " << *L << "\n"; } } while (false); | |||
5792 | return; | |||
5793 | } | |||
5794 | ||||
5795 | // Start collecting data and preparing for the solver. | |||
5796 | // If number of registers is not the major cost, we cannot benefit from the | |||
5797 | // current profitable chain optimization which is based on number of | |||
5798 | // registers. | |||
5799 | // FIXME: add profitable chain optimization for other kinds major cost, for | |||
5800 | // example number of instructions. | |||
5801 | if (TTI.isNumRegsMajorCostOfLSR() || StressIVChain) | |||
5802 | CollectChains(); | |||
5803 | CollectInterestingTypesAndFactors(); | |||
5804 | CollectFixupsAndInitialFormulae(); | |||
5805 | CollectLoopInvariantFixupsAndFormulae(); | |||
5806 | ||||
5807 | if (Uses.empty()) | |||
5808 | return; | |||
5809 | ||||
5810 | LLVM_DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "LSR found " << Uses .size() << " uses:\n"; print_uses(dbgs()); } } while (false ) | |||
5811 | print_uses(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "LSR found " << Uses .size() << " uses:\n"; print_uses(dbgs()); } } while (false ); | |||
5812 | ||||
5813 | // Now use the reuse data to generate a bunch of interesting ways | |||
5814 | // to formulate the values needed for the uses. | |||
5815 | GenerateAllReuseFormulae(); | |||
5816 | ||||
5817 | FilterOutUndesirableDedicatedRegisters(); | |||
5818 | NarrowSearchSpaceUsingHeuristics(); | |||
5819 | ||||
5820 | SmallVector<const Formula *, 8> Solution; | |||
5821 | Solve(Solution); | |||
5822 | ||||
5823 | // Release memory that is no longer needed. | |||
5824 | Factors.clear(); | |||
5825 | Types.clear(); | |||
5826 | RegUses.clear(); | |||
5827 | ||||
5828 | if (Solution.empty()) | |||
5829 | return; | |||
5830 | ||||
5831 | #ifndef NDEBUG | |||
5832 | // Formulae should be legal. | |||
5833 | for (const LSRUse &LU : Uses) { | |||
5834 | for (const Formula &F : LU.Formulae) | |||
5835 | assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,(static_cast <bool> (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset , LU.Kind, LU.AccessTy, F) && "Illegal formula generated!" ) ? void (0) : __assert_fail ("isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) && \"Illegal formula generated!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5836, __extension__ __PRETTY_FUNCTION__)) | |||
5836 | F) && "Illegal formula generated!")(static_cast <bool> (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset , LU.Kind, LU.AccessTy, F) && "Illegal formula generated!" ) ? void (0) : __assert_fail ("isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) && \"Illegal formula generated!\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 5836, __extension__ __PRETTY_FUNCTION__)); | |||
5837 | }; | |||
5838 | #endif | |||
5839 | ||||
5840 | // Now that we've decided what we want, make it so. | |||
5841 | ImplementSolution(Solution); | |||
5842 | } | |||
5843 | ||||
5844 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
5845 | void LSRInstance::print_factors_and_types(raw_ostream &OS) const { | |||
5846 | if (Factors.empty() && Types.empty()) return; | |||
5847 | ||||
5848 | OS << "LSR has identified the following interesting factors and types: "; | |||
5849 | bool First = true; | |||
5850 | ||||
5851 | for (int64_t Factor : Factors) { | |||
5852 | if (!First) OS << ", "; | |||
5853 | First = false; | |||
5854 | OS << '*' << Factor; | |||
5855 | } | |||
5856 | ||||
5857 | for (Type *Ty : Types) { | |||
5858 | if (!First) OS << ", "; | |||
5859 | First = false; | |||
5860 | OS << '(' << *Ty << ')'; | |||
5861 | } | |||
5862 | OS << '\n'; | |||
5863 | } | |||
5864 | ||||
5865 | void LSRInstance::print_fixups(raw_ostream &OS) const { | |||
5866 | OS << "LSR is examining the following fixup sites:\n"; | |||
5867 | for (const LSRUse &LU : Uses) | |||
5868 | for (const LSRFixup &LF : LU.Fixups) { | |||
5869 | dbgs() << " "; | |||
5870 | LF.print(OS); | |||
5871 | OS << '\n'; | |||
5872 | } | |||
5873 | } | |||
5874 | ||||
5875 | void LSRInstance::print_uses(raw_ostream &OS) const { | |||
5876 | OS << "LSR is examining the following uses:\n"; | |||
5877 | for (const LSRUse &LU : Uses) { | |||
5878 | dbgs() << " "; | |||
5879 | LU.print(OS); | |||
5880 | OS << '\n'; | |||
5881 | for (const Formula &F : LU.Formulae) { | |||
5882 | OS << " "; | |||
5883 | F.print(OS); | |||
5884 | OS << '\n'; | |||
5885 | } | |||
5886 | } | |||
5887 | } | |||
5888 | ||||
5889 | void LSRInstance::print(raw_ostream &OS) const { | |||
5890 | print_factors_and_types(OS); | |||
5891 | print_fixups(OS); | |||
5892 | print_uses(OS); | |||
5893 | } | |||
5894 | ||||
5895 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void LSRInstance::dump() const { | |||
5896 | print(errs()); errs() << '\n'; | |||
5897 | } | |||
5898 | #endif | |||
5899 | ||||
5900 | namespace { | |||
5901 | ||||
5902 | class LoopStrengthReduce : public LoopPass { | |||
5903 | public: | |||
5904 | static char ID; // Pass ID, replacement for typeid | |||
5905 | ||||
5906 | LoopStrengthReduce(); | |||
5907 | ||||
5908 | private: | |||
5909 | bool runOnLoop(Loop *L, LPPassManager &LPM) override; | |||
5910 | void getAnalysisUsage(AnalysisUsage &AU) const override; | |||
5911 | }; | |||
5912 | ||||
5913 | } // end anonymous namespace | |||
5914 | ||||
5915 | LoopStrengthReduce::LoopStrengthReduce() : LoopPass(ID) { | |||
5916 | initializeLoopStrengthReducePass(*PassRegistry::getPassRegistry()); | |||
5917 | } | |||
5918 | ||||
5919 | void LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const { | |||
5920 | // We split critical edges, so we change the CFG. However, we do update | |||
5921 | // many analyses if they are around. | |||
5922 | AU.addPreservedID(LoopSimplifyID); | |||
5923 | ||||
5924 | AU.addRequired<LoopInfoWrapperPass>(); | |||
5925 | AU.addPreserved<LoopInfoWrapperPass>(); | |||
5926 | AU.addRequiredID(LoopSimplifyID); | |||
5927 | AU.addRequired<DominatorTreeWrapperPass>(); | |||
5928 | AU.addPreserved<DominatorTreeWrapperPass>(); | |||
5929 | AU.addRequired<ScalarEvolutionWrapperPass>(); | |||
5930 | AU.addPreserved<ScalarEvolutionWrapperPass>(); | |||
5931 | AU.addRequired<AssumptionCacheTracker>(); | |||
5932 | AU.addRequired<TargetLibraryInfoWrapperPass>(); | |||
5933 | // Requiring LoopSimplify a second time here prevents IVUsers from running | |||
5934 | // twice, since LoopSimplify was invalidated by running ScalarEvolution. | |||
5935 | AU.addRequiredID(LoopSimplifyID); | |||
5936 | AU.addRequired<IVUsersWrapperPass>(); | |||
5937 | AU.addPreserved<IVUsersWrapperPass>(); | |||
5938 | AU.addRequired<TargetTransformInfoWrapperPass>(); | |||
5939 | AU.addPreserved<MemorySSAWrapperPass>(); | |||
5940 | } | |||
5941 | ||||
5942 | namespace { | |||
5943 | ||||
5944 | /// Enables more convenient iteration over a DWARF expression vector. | |||
5945 | static iterator_range<llvm::DIExpression::expr_op_iterator> | |||
5946 | ToDwarfOpIter(SmallVectorImpl<uint64_t> &Expr) { | |||
5947 | llvm::DIExpression::expr_op_iterator Begin = | |||
5948 | llvm::DIExpression::expr_op_iterator(Expr.begin()); | |||
5949 | llvm::DIExpression::expr_op_iterator End = | |||
5950 | llvm::DIExpression::expr_op_iterator(Expr.end()); | |||
5951 | return {Begin, End}; | |||
5952 | } | |||
5953 | ||||
5954 | struct SCEVDbgValueBuilder { | |||
5955 | SCEVDbgValueBuilder() = default; | |||
5956 | SCEVDbgValueBuilder(const SCEVDbgValueBuilder &Base) { clone(Base); } | |||
5957 | ||||
5958 | void clone(const SCEVDbgValueBuilder &Base) { | |||
5959 | LocationOps = Base.LocationOps; | |||
5960 | Expr = Base.Expr; | |||
5961 | } | |||
5962 | ||||
5963 | void clear() { | |||
5964 | LocationOps.clear(); | |||
5965 | Expr.clear(); | |||
5966 | } | |||
5967 | ||||
5968 | /// The DIExpression as we translate the SCEV. | |||
5969 | SmallVector<uint64_t, 6> Expr; | |||
5970 | /// The location ops of the DIExpression. | |||
5971 | SmallVector<Value *, 2> LocationOps; | |||
5972 | ||||
5973 | void pushOperator(uint64_t Op) { Expr.push_back(Op); } | |||
5974 | void pushUInt(uint64_t Operand) { Expr.push_back(Operand); } | |||
5975 | ||||
5976 | /// Add a DW_OP_LLVM_arg to the expression, followed by the index of the value | |||
5977 | /// in the set of values referenced by the expression. | |||
5978 | void pushLocation(llvm::Value *V) { | |||
5979 | Expr.push_back(llvm::dwarf::DW_OP_LLVM_arg); | |||
5980 | auto *It = llvm::find(LocationOps, V); | |||
5981 | unsigned ArgIndex = 0; | |||
5982 | if (It != LocationOps.end()) { | |||
5983 | ArgIndex = std::distance(LocationOps.begin(), It); | |||
5984 | } else { | |||
5985 | ArgIndex = LocationOps.size(); | |||
5986 | LocationOps.push_back(V); | |||
5987 | } | |||
5988 | Expr.push_back(ArgIndex); | |||
5989 | } | |||
5990 | ||||
5991 | void pushValue(const SCEVUnknown *U) { | |||
5992 | llvm::Value *V = cast<SCEVUnknown>(U)->getValue(); | |||
5993 | pushLocation(V); | |||
5994 | } | |||
5995 | ||||
5996 | bool pushConst(const SCEVConstant *C) { | |||
5997 | if (C->getAPInt().getMinSignedBits() > 64) | |||
5998 | return false; | |||
5999 | Expr.push_back(llvm::dwarf::DW_OP_consts); | |||
6000 | Expr.push_back(C->getAPInt().getSExtValue()); | |||
6001 | return true; | |||
6002 | } | |||
6003 | ||||
6004 | // Iterating the expression as DWARF ops is convenient when updating | |||
6005 | // DWARF_OP_LLVM_args. | |||
6006 | iterator_range<llvm::DIExpression::expr_op_iterator> expr_ops() { | |||
6007 | return ToDwarfOpIter(Expr); | |||
6008 | } | |||
6009 | ||||
6010 | /// Several SCEV types are sequences of the same arithmetic operator applied | |||
6011 | /// to constants and values that may be extended or truncated. | |||
6012 | bool pushArithmeticExpr(const llvm::SCEVCommutativeExpr *CommExpr, | |||
6013 | uint64_t DwarfOp) { | |||
6014 | assert((isa<llvm::SCEVAddExpr>(CommExpr) || isa<SCEVMulExpr>(CommExpr)) &&(static_cast <bool> ((isa<llvm::SCEVAddExpr>(CommExpr ) || isa<SCEVMulExpr>(CommExpr)) && "Expected arithmetic SCEV type" ) ? void (0) : __assert_fail ("(isa<llvm::SCEVAddExpr>(CommExpr) || isa<SCEVMulExpr>(CommExpr)) && \"Expected arithmetic SCEV type\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6015, __extension__ __PRETTY_FUNCTION__)) | |||
6015 | "Expected arithmetic SCEV type")(static_cast <bool> ((isa<llvm::SCEVAddExpr>(CommExpr ) || isa<SCEVMulExpr>(CommExpr)) && "Expected arithmetic SCEV type" ) ? void (0) : __assert_fail ("(isa<llvm::SCEVAddExpr>(CommExpr) || isa<SCEVMulExpr>(CommExpr)) && \"Expected arithmetic SCEV type\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6015, __extension__ __PRETTY_FUNCTION__)); | |||
6016 | bool Success = true; | |||
6017 | unsigned EmitOperator = 0; | |||
6018 | for (const auto &Op : CommExpr->operands()) { | |||
6019 | Success &= pushSCEV(Op); | |||
6020 | ||||
6021 | if (EmitOperator >= 1) | |||
6022 | pushOperator(DwarfOp); | |||
6023 | ++EmitOperator; | |||
6024 | } | |||
6025 | return Success; | |||
6026 | } | |||
6027 | ||||
6028 | // TODO: Identify and omit noop casts. | |||
6029 | bool pushCast(const llvm::SCEVCastExpr *C, bool IsSigned) { | |||
6030 | const llvm::SCEV *Inner = C->getOperand(0); | |||
6031 | const llvm::Type *Type = C->getType(); | |||
6032 | uint64_t ToWidth = Type->getIntegerBitWidth(); | |||
6033 | bool Success = pushSCEV(Inner); | |||
6034 | uint64_t CastOps[] = {dwarf::DW_OP_LLVM_convert, ToWidth, | |||
6035 | IsSigned ? llvm::dwarf::DW_ATE_signed | |||
6036 | : llvm::dwarf::DW_ATE_unsigned}; | |||
6037 | for (const auto &Op : CastOps) | |||
6038 | pushOperator(Op); | |||
6039 | return Success; | |||
6040 | } | |||
6041 | ||||
6042 | // TODO: MinMax - although these haven't been encountered in the test suite. | |||
6043 | bool pushSCEV(const llvm::SCEV *S) { | |||
6044 | bool Success = true; | |||
6045 | if (const SCEVConstant *StartInt = dyn_cast<SCEVConstant>(S)) { | |||
6046 | Success &= pushConst(StartInt); | |||
6047 | ||||
6048 | } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) { | |||
6049 | if (!U->getValue()) | |||
6050 | return false; | |||
6051 | pushLocation(U->getValue()); | |||
6052 | ||||
6053 | } else if (const SCEVMulExpr *MulRec = dyn_cast<SCEVMulExpr>(S)) { | |||
6054 | Success &= pushArithmeticExpr(MulRec, llvm::dwarf::DW_OP_mul); | |||
6055 | ||||
6056 | } else if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) { | |||
6057 | Success &= pushSCEV(UDiv->getLHS()); | |||
6058 | Success &= pushSCEV(UDiv->getRHS()); | |||
6059 | pushOperator(llvm::dwarf::DW_OP_div); | |||
6060 | ||||
6061 | } else if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(S)) { | |||
6062 | // Assert if a new and unknown SCEVCastEXpr type is encountered. | |||
6063 | assert((isa<SCEVZeroExtendExpr>(Cast) || isa<SCEVTruncateExpr>(Cast) ||(static_cast <bool> ((isa<SCEVZeroExtendExpr>(Cast ) || isa<SCEVTruncateExpr>(Cast) || isa<SCEVPtrToIntExpr >(Cast) || isa<SCEVSignExtendExpr>(Cast)) && "Unexpected cast type in SCEV.") ? void (0) : __assert_fail ( "(isa<SCEVZeroExtendExpr>(Cast) || isa<SCEVTruncateExpr>(Cast) || isa<SCEVPtrToIntExpr>(Cast) || isa<SCEVSignExtendExpr>(Cast)) && \"Unexpected cast type in SCEV.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6065, __extension__ __PRETTY_FUNCTION__)) | |||
6064 | isa<SCEVPtrToIntExpr>(Cast) || isa<SCEVSignExtendExpr>(Cast)) &&(static_cast <bool> ((isa<SCEVZeroExtendExpr>(Cast ) || isa<SCEVTruncateExpr>(Cast) || isa<SCEVPtrToIntExpr >(Cast) || isa<SCEVSignExtendExpr>(Cast)) && "Unexpected cast type in SCEV.") ? void (0) : __assert_fail ( "(isa<SCEVZeroExtendExpr>(Cast) || isa<SCEVTruncateExpr>(Cast) || isa<SCEVPtrToIntExpr>(Cast) || isa<SCEVSignExtendExpr>(Cast)) && \"Unexpected cast type in SCEV.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6065, __extension__ __PRETTY_FUNCTION__)) | |||
6065 | "Unexpected cast type in SCEV.")(static_cast <bool> ((isa<SCEVZeroExtendExpr>(Cast ) || isa<SCEVTruncateExpr>(Cast) || isa<SCEVPtrToIntExpr >(Cast) || isa<SCEVSignExtendExpr>(Cast)) && "Unexpected cast type in SCEV.") ? void (0) : __assert_fail ( "(isa<SCEVZeroExtendExpr>(Cast) || isa<SCEVTruncateExpr>(Cast) || isa<SCEVPtrToIntExpr>(Cast) || isa<SCEVSignExtendExpr>(Cast)) && \"Unexpected cast type in SCEV.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6065, __extension__ __PRETTY_FUNCTION__)); | |||
6066 | Success &= pushCast(Cast, (isa<SCEVSignExtendExpr>(Cast))); | |||
6067 | ||||
6068 | } else if (const SCEVAddExpr *AddExpr = dyn_cast<SCEVAddExpr>(S)) { | |||
6069 | Success &= pushArithmeticExpr(AddExpr, llvm::dwarf::DW_OP_plus); | |||
6070 | ||||
6071 | } else if (isa<SCEVAddRecExpr>(S)) { | |||
6072 | // Nested SCEVAddRecExpr are generated by nested loops and are currently | |||
6073 | // unsupported. | |||
6074 | return false; | |||
6075 | ||||
6076 | } else { | |||
6077 | return false; | |||
6078 | } | |||
6079 | return Success; | |||
6080 | } | |||
6081 | ||||
6082 | /// Return true if the combination of arithmetic operator and underlying | |||
6083 | /// SCEV constant value is an identity function. | |||
6084 | bool isIdentityFunction(uint64_t Op, const SCEV *S) { | |||
6085 | if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) { | |||
6086 | if (C->getAPInt().getMinSignedBits() > 64) | |||
6087 | return false; | |||
6088 | int64_t I = C->getAPInt().getSExtValue(); | |||
6089 | switch (Op) { | |||
6090 | case llvm::dwarf::DW_OP_plus: | |||
6091 | case llvm::dwarf::DW_OP_minus: | |||
6092 | return I == 0; | |||
6093 | case llvm::dwarf::DW_OP_mul: | |||
6094 | case llvm::dwarf::DW_OP_div: | |||
6095 | return I == 1; | |||
6096 | } | |||
6097 | } | |||
6098 | return false; | |||
6099 | } | |||
6100 | ||||
6101 | /// Convert a SCEV of a value to a DIExpression that is pushed onto the | |||
6102 | /// builder's expression stack. The stack should already contain an | |||
6103 | /// expression for the iteration count, so that it can be multiplied by | |||
6104 | /// the stride and added to the start. | |||
6105 | /// Components of the expression are omitted if they are an identity function. | |||
6106 | /// Chain (non-affine) SCEVs are not supported. | |||
6107 | bool SCEVToValueExpr(const llvm::SCEVAddRecExpr &SAR, ScalarEvolution &SE) { | |||
6108 | assert(SAR.isAffine() && "Expected affine SCEV")(static_cast <bool> (SAR.isAffine() && "Expected affine SCEV" ) ? void (0) : __assert_fail ("SAR.isAffine() && \"Expected affine SCEV\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6108, __extension__ __PRETTY_FUNCTION__)); | |||
6109 | // TODO: Is this check needed? | |||
6110 | if (isa<SCEVAddRecExpr>(SAR.getStart())) | |||
6111 | return false; | |||
6112 | ||||
6113 | const SCEV *Start = SAR.getStart(); | |||
6114 | const SCEV *Stride = SAR.getStepRecurrence(SE); | |||
6115 | ||||
6116 | // Skip pushing arithmetic noops. | |||
6117 | if (!isIdentityFunction(llvm::dwarf::DW_OP_mul, Stride)) { | |||
6118 | if (!pushSCEV(Stride)) | |||
6119 | return false; | |||
6120 | pushOperator(llvm::dwarf::DW_OP_mul); | |||
6121 | } | |||
6122 | if (!isIdentityFunction(llvm::dwarf::DW_OP_plus, Start)) { | |||
6123 | if (!pushSCEV(Start)) | |||
6124 | return false; | |||
6125 | pushOperator(llvm::dwarf::DW_OP_plus); | |||
6126 | } | |||
6127 | return true; | |||
6128 | } | |||
6129 | ||||
6130 | /// Create an expression that is an offset from a value (usually the IV). | |||
6131 | void createOffsetExpr(int64_t Offset, Value *OffsetValue) { | |||
6132 | pushLocation(OffsetValue); | |||
6133 | DIExpression::appendOffset(Expr, Offset); | |||
6134 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: Generated IV offset expression. Offset: " << std::to_string(Offset) << "\n"; } } while (false ) | |||
6135 | dbgs() << "scev-salvage: Generated IV offset expression. Offset: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: Generated IV offset expression. Offset: " << std::to_string(Offset) << "\n"; } } while (false ) | |||
6136 | << std::to_string(Offset) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: Generated IV offset expression. Offset: " << std::to_string(Offset) << "\n"; } } while (false ); | |||
6137 | } | |||
6138 | ||||
6139 | /// Combine a translation of the SCEV and the IV to create an expression that | |||
6140 | /// recovers a location's value. | |||
6141 | /// returns true if an expression was created. | |||
6142 | bool createIterCountExpr(const SCEV *S, | |||
6143 | const SCEVDbgValueBuilder &IterationCount, | |||
6144 | ScalarEvolution &SE) { | |||
6145 | // SCEVs for SSA values are most frquently of the form | |||
6146 | // {start,+,stride}, but sometimes they are ({start,+,stride} + %a + ..). | |||
6147 | // This is because %a is a PHI node that is not the IV. However, these | |||
6148 | // SCEVs have not been observed to result in debuginfo-lossy optimisations, | |||
6149 | // so its not expected this point will be reached. | |||
6150 | if (!isa<SCEVAddRecExpr>(S)) | |||
6151 | return false; | |||
6152 | ||||
6153 | LLVM_DEBUG(dbgs() << "scev-salvage: Location to salvage SCEV: " << *Sdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: Location to salvage SCEV: " << *S << '\n'; } } while (false) | |||
6154 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: Location to salvage SCEV: " << *S << '\n'; } } while (false); | |||
6155 | ||||
6156 | const auto *Rec = cast<SCEVAddRecExpr>(S); | |||
6157 | if (!Rec->isAffine()) | |||
6158 | return false; | |||
6159 | ||||
6160 | if (S->getExpressionSize() > MaxSCEVSalvageExpressionSize) | |||
6161 | return false; | |||
6162 | ||||
6163 | // Initialise a new builder with the iteration count expression. In | |||
6164 | // combination with the value's SCEV this enables recovery. | |||
6165 | clone(IterationCount); | |||
6166 | if (!SCEVToValueExpr(*Rec, SE)) | |||
6167 | return false; | |||
6168 | ||||
6169 | return true; | |||
6170 | } | |||
6171 | ||||
6172 | /// Convert a SCEV of a value to a DIExpression that is pushed onto the | |||
6173 | /// builder's expression stack. The stack should already contain an | |||
6174 | /// expression for the iteration count, so that it can be multiplied by | |||
6175 | /// the stride and added to the start. | |||
6176 | /// Components of the expression are omitted if they are an identity function. | |||
6177 | bool SCEVToIterCountExpr(const llvm::SCEVAddRecExpr &SAR, | |||
6178 | ScalarEvolution &SE) { | |||
6179 | assert(SAR.isAffine() && "Expected affine SCEV")(static_cast <bool> (SAR.isAffine() && "Expected affine SCEV" ) ? void (0) : __assert_fail ("SAR.isAffine() && \"Expected affine SCEV\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6179, __extension__ __PRETTY_FUNCTION__)); | |||
6180 | if (isa<SCEVAddRecExpr>(SAR.getStart())) { | |||
6181 | LLVM_DEBUG(dbgs() << "scev-salvage: IV SCEV. Unsupported nested AddRec: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: IV SCEV. Unsupported nested AddRec: " << SAR << '\n'; } } while (false) | |||
6182 | << SAR << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: IV SCEV. Unsupported nested AddRec: " << SAR << '\n'; } } while (false); | |||
6183 | return false; | |||
6184 | } | |||
6185 | const SCEV *Start = SAR.getStart(); | |||
6186 | const SCEV *Stride = SAR.getStepRecurrence(SE); | |||
6187 | ||||
6188 | // Skip pushing arithmetic noops. | |||
6189 | if (!isIdentityFunction(llvm::dwarf::DW_OP_minus, Start)) { | |||
6190 | if (!pushSCEV(Start)) | |||
6191 | return false; | |||
6192 | pushOperator(llvm::dwarf::DW_OP_minus); | |||
6193 | } | |||
6194 | if (!isIdentityFunction(llvm::dwarf::DW_OP_div, Stride)) { | |||
6195 | if (!pushSCEV(Stride)) | |||
6196 | return false; | |||
6197 | pushOperator(llvm::dwarf::DW_OP_div); | |||
6198 | } | |||
6199 | return true; | |||
6200 | } | |||
6201 | ||||
6202 | // Append the current expression and locations to a location list and an | |||
6203 | // expression list. Modify the DW_OP_LLVM_arg indexes to account for | |||
6204 | // the locations already present in the destination list. | |||
6205 | void appendToVectors(SmallVectorImpl<uint64_t> &DestExpr, | |||
6206 | SmallVectorImpl<Value *> &DestLocations) { | |||
6207 | assert(!DestLocations.empty() &&(static_cast <bool> (!DestLocations.empty() && "Expected the locations vector to contain the IV" ) ? void (0) : __assert_fail ("!DestLocations.empty() && \"Expected the locations vector to contain the IV\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6208, __extension__ __PRETTY_FUNCTION__)) | |||
6208 | "Expected the locations vector to contain the IV")(static_cast <bool> (!DestLocations.empty() && "Expected the locations vector to contain the IV" ) ? void (0) : __assert_fail ("!DestLocations.empty() && \"Expected the locations vector to contain the IV\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6208, __extension__ __PRETTY_FUNCTION__)); | |||
6209 | // The DWARF_OP_LLVM_arg arguments of the expression being appended must be | |||
6210 | // modified to account for the locations already in the destination vector. | |||
6211 | // All builders contain the IV as the first location op. | |||
6212 | assert(!LocationOps.empty() &&(static_cast <bool> (!LocationOps.empty() && "Expected the location ops to contain the IV." ) ? void (0) : __assert_fail ("!LocationOps.empty() && \"Expected the location ops to contain the IV.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6213, __extension__ __PRETTY_FUNCTION__)) | |||
6213 | "Expected the location ops to contain the IV.")(static_cast <bool> (!LocationOps.empty() && "Expected the location ops to contain the IV." ) ? void (0) : __assert_fail ("!LocationOps.empty() && \"Expected the location ops to contain the IV.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6213, __extension__ __PRETTY_FUNCTION__)); | |||
6214 | // DestIndexMap[n] contains the index in DestLocations for the nth | |||
6215 | // location in this SCEVDbgValueBuilder. | |||
6216 | SmallVector<uint64_t, 2> DestIndexMap; | |||
6217 | for (const auto &Op : LocationOps) { | |||
6218 | auto It = find(DestLocations, Op); | |||
6219 | if (It != DestLocations.end()) { | |||
6220 | // Location already exists in DestLocations, reuse existing ArgIndex. | |||
6221 | DestIndexMap.push_back(std::distance(DestLocations.begin(), It)); | |||
6222 | continue; | |||
6223 | } | |||
6224 | // Location is not in DestLocations, add it. | |||
6225 | DestIndexMap.push_back(DestLocations.size()); | |||
6226 | DestLocations.push_back(Op); | |||
6227 | } | |||
6228 | ||||
6229 | for (const auto &Op : expr_ops()) { | |||
6230 | if (Op.getOp() != dwarf::DW_OP_LLVM_arg) { | |||
6231 | Op.appendToVector(DestExpr); | |||
6232 | continue; | |||
6233 | } | |||
6234 | ||||
6235 | DestExpr.push_back(dwarf::DW_OP_LLVM_arg); | |||
6236 | // `DW_OP_LLVM_arg n` represents the nth LocationOp in this SCEV, | |||
6237 | // DestIndexMap[n] contains its new index in DestLocations. | |||
6238 | uint64_t NewIndex = DestIndexMap[Op.getArg(0)]; | |||
6239 | DestExpr.push_back(NewIndex); | |||
6240 | } | |||
6241 | } | |||
6242 | }; | |||
6243 | ||||
6244 | /// Holds all the required data to salvage a dbg.value using the pre-LSR SCEVs | |||
6245 | /// and DIExpression. | |||
6246 | struct DVIRecoveryRec { | |||
6247 | DVIRecoveryRec(DbgValueInst *DbgValue) | |||
6248 | : DVI(DbgValue), Expr(DbgValue->getExpression()), | |||
6249 | HadLocationArgList(false) {} | |||
6250 | ||||
6251 | DbgValueInst *DVI; | |||
6252 | DIExpression *Expr; | |||
6253 | bool HadLocationArgList; | |||
6254 | SmallVector<WeakVH, 2> LocationOps; | |||
6255 | SmallVector<const llvm::SCEV *, 2> SCEVs; | |||
6256 | SmallVector<std::unique_ptr<SCEVDbgValueBuilder>, 2> RecoveryExprs; | |||
6257 | ||||
6258 | void clear() { | |||
6259 | for (auto &RE : RecoveryExprs) | |||
6260 | RE.reset(); | |||
6261 | RecoveryExprs.clear(); | |||
6262 | } | |||
6263 | ||||
6264 | ~DVIRecoveryRec() { clear(); } | |||
6265 | }; | |||
6266 | } // namespace | |||
6267 | ||||
6268 | /// Returns the total number of DW_OP_llvm_arg operands in the expression. | |||
6269 | /// This helps in determining if a DIArglist is necessary or can be omitted from | |||
6270 | /// the dbg.value. | |||
6271 | static unsigned numLLVMArgOps(SmallVectorImpl<uint64_t> &Expr) { | |||
6272 | auto expr_ops = ToDwarfOpIter(Expr); | |||
6273 | unsigned Count = 0; | |||
6274 | for (auto Op : expr_ops) | |||
6275 | if (Op.getOp() == dwarf::DW_OP_LLVM_arg) | |||
6276 | Count++; | |||
6277 | return Count; | |||
6278 | } | |||
6279 | ||||
6280 | /// Overwrites DVI with the location and Ops as the DIExpression. This will | |||
6281 | /// create an invalid expression if Ops has any dwarf::DW_OP_llvm_arg operands, | |||
6282 | /// because a DIArglist is not created for the first argument of the dbg.value. | |||
6283 | static void updateDVIWithLocation(DbgValueInst &DVI, Value *Location, | |||
6284 | SmallVectorImpl<uint64_t> &Ops) { | |||
6285 | assert((static_cast <bool> (numLLVMArgOps(Ops) == 0 && "Expected expression that does not contain any DW_OP_llvm_arg operands." ) ? void (0) : __assert_fail ("numLLVMArgOps(Ops) == 0 && \"Expected expression that does not contain any DW_OP_llvm_arg operands.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6287, __extension__ __PRETTY_FUNCTION__)) | |||
6286 | numLLVMArgOps(Ops) == 0 &&(static_cast <bool> (numLLVMArgOps(Ops) == 0 && "Expected expression that does not contain any DW_OP_llvm_arg operands." ) ? void (0) : __assert_fail ("numLLVMArgOps(Ops) == 0 && \"Expected expression that does not contain any DW_OP_llvm_arg operands.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6287, __extension__ __PRETTY_FUNCTION__)) | |||
6287 | "Expected expression that does not contain any DW_OP_llvm_arg operands.")(static_cast <bool> (numLLVMArgOps(Ops) == 0 && "Expected expression that does not contain any DW_OP_llvm_arg operands." ) ? void (0) : __assert_fail ("numLLVMArgOps(Ops) == 0 && \"Expected expression that does not contain any DW_OP_llvm_arg operands.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6287, __extension__ __PRETTY_FUNCTION__)); | |||
6288 | DVI.setRawLocation(ValueAsMetadata::get(Location)); | |||
6289 | DVI.setExpression(DIExpression::get(DVI.getContext(), Ops)); | |||
6290 | } | |||
6291 | ||||
6292 | /// Overwrite DVI with locations placed into a DIArglist. | |||
6293 | static void updateDVIWithLocations(DbgValueInst &DVI, | |||
6294 | SmallVectorImpl<Value *> &Locations, | |||
6295 | SmallVectorImpl<uint64_t> &Ops) { | |||
6296 | assert(numLLVMArgOps(Ops) != 0 &&(static_cast <bool> (numLLVMArgOps(Ops) != 0 && "Expected expression that references DIArglist locations using " "DW_OP_llvm_arg operands.") ? void (0) : __assert_fail ("numLLVMArgOps(Ops) != 0 && \"Expected expression that references DIArglist locations using \" \"DW_OP_llvm_arg operands.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6298, __extension__ __PRETTY_FUNCTION__)) | |||
6297 | "Expected expression that references DIArglist locations using "(static_cast <bool> (numLLVMArgOps(Ops) != 0 && "Expected expression that references DIArglist locations using " "DW_OP_llvm_arg operands.") ? void (0) : __assert_fail ("numLLVMArgOps(Ops) != 0 && \"Expected expression that references DIArglist locations using \" \"DW_OP_llvm_arg operands.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6298, __extension__ __PRETTY_FUNCTION__)) | |||
6298 | "DW_OP_llvm_arg operands.")(static_cast <bool> (numLLVMArgOps(Ops) != 0 && "Expected expression that references DIArglist locations using " "DW_OP_llvm_arg operands.") ? void (0) : __assert_fail ("numLLVMArgOps(Ops) != 0 && \"Expected expression that references DIArglist locations using \" \"DW_OP_llvm_arg operands.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6298, __extension__ __PRETTY_FUNCTION__)); | |||
6299 | SmallVector<ValueAsMetadata *, 3> MetadataLocs; | |||
6300 | for (Value *V : Locations) | |||
6301 | MetadataLocs.push_back(ValueAsMetadata::get(V)); | |||
6302 | auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs); | |||
6303 | DVI.setRawLocation(llvm::DIArgList::get(DVI.getContext(), ValArrayRef)); | |||
6304 | DVI.setExpression(DIExpression::get(DVI.getContext(), Ops)); | |||
6305 | } | |||
6306 | ||||
6307 | /// Write the new expression and new location ops for the dbg.value. If possible | |||
6308 | /// reduce the szie of the dbg.value intrinsic by omitting DIArglist. This | |||
6309 | /// can be omitted if: | |||
6310 | /// 1. There is only a single location, refenced by a single DW_OP_llvm_arg. | |||
6311 | /// 2. The DW_OP_LLVM_arg is the first operand in the expression. | |||
6312 | static void UpdateDbgValueInst(DVIRecoveryRec &DVIRec, | |||
6313 | SmallVectorImpl<Value *> &NewLocationOps, | |||
6314 | SmallVectorImpl<uint64_t> &NewExpr) { | |||
6315 | unsigned NumLLVMArgs = numLLVMArgOps(NewExpr); | |||
6316 | if (NumLLVMArgs == 0) { | |||
6317 | // Location assumed to be on the stack. | |||
6318 | updateDVIWithLocation(*DVIRec.DVI, NewLocationOps[0], NewExpr); | |||
6319 | } else if (NumLLVMArgs == 1 && NewExpr[0] == dwarf::DW_OP_LLVM_arg) { | |||
6320 | // There is only a single DW_OP_llvm_arg at the start of the expression, | |||
6321 | // so it can be omitted along with DIArglist. | |||
6322 | assert(NewExpr[1] == 0 &&(static_cast <bool> (NewExpr[1] == 0 && "Lone LLVM_arg in a DIExpression should refer to location-op 0." ) ? void (0) : __assert_fail ("NewExpr[1] == 0 && \"Lone LLVM_arg in a DIExpression should refer to location-op 0.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6323, __extension__ __PRETTY_FUNCTION__)) | |||
6323 | "Lone LLVM_arg in a DIExpression should refer to location-op 0.")(static_cast <bool> (NewExpr[1] == 0 && "Lone LLVM_arg in a DIExpression should refer to location-op 0." ) ? void (0) : __assert_fail ("NewExpr[1] == 0 && \"Lone LLVM_arg in a DIExpression should refer to location-op 0.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6323, __extension__ __PRETTY_FUNCTION__)); | |||
6324 | llvm::SmallVector<uint64_t, 6> ShortenedOps(llvm::drop_begin(NewExpr, 2)); | |||
6325 | updateDVIWithLocation(*DVIRec.DVI, NewLocationOps[0], ShortenedOps); | |||
6326 | } else { | |||
6327 | // Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary. | |||
6328 | updateDVIWithLocations(*DVIRec.DVI, NewLocationOps, NewExpr); | |||
6329 | } | |||
6330 | ||||
6331 | // If the DIExpression was previously empty then add the stack terminator. | |||
6332 | // Non-empty expressions have only had elements inserted into them and so the | |||
6333 | // terminator should already be present e.g. stack_value or fragment. | |||
6334 | DIExpression *SalvageExpr = DVIRec.DVI->getExpression(); | |||
6335 | if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) { | |||
6336 | SalvageExpr = DIExpression::append(SalvageExpr, {dwarf::DW_OP_stack_value}); | |||
6337 | DVIRec.DVI->setExpression(SalvageExpr); | |||
6338 | } | |||
6339 | } | |||
6340 | ||||
6341 | /// Cached location ops may be erased during LSR, in which case an undef is | |||
6342 | /// required when restoring from the cache. The type of that location is no | |||
6343 | /// longer available, so just use int8. The undef will be replaced by one or | |||
6344 | /// more locations later when a SCEVDbgValueBuilder selects alternative | |||
6345 | /// locations to use for the salvage. | |||
6346 | static Value *getValueOrUndef(WeakVH &VH, LLVMContext &C) { | |||
6347 | return (VH) ? VH : UndefValue::get(llvm::Type::getInt8Ty(C)); | |||
6348 | } | |||
6349 | ||||
6350 | /// Restore the DVI's pre-LSR arguments. Substitute undef for any erased values. | |||
6351 | static void restorePreTransformState(DVIRecoveryRec &DVIRec) { | |||
6352 | LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n" << "scev-salvage: post-LSR: " << *DVIRec.DVI << '\n'; } } while (false) | |||
6353 | << "scev-salvage: post-LSR: " << *DVIRec.DVI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n" << "scev-salvage: post-LSR: " << *DVIRec.DVI << '\n'; } } while (false); | |||
6354 | assert(DVIRec.Expr && "Expected an expression")(static_cast <bool> (DVIRec.Expr && "Expected an expression" ) ? void (0) : __assert_fail ("DVIRec.Expr && \"Expected an expression\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6354, __extension__ __PRETTY_FUNCTION__)); | |||
6355 | DVIRec.DVI->setExpression(DVIRec.Expr); | |||
6356 | ||||
6357 | // Even a single location-op may be inside a DIArgList and referenced with | |||
6358 | // DW_OP_LLVM_arg, which is valid only with a DIArgList. | |||
6359 | if (!DVIRec.HadLocationArgList) { | |||
6360 | assert(DVIRec.LocationOps.size() == 1 &&(static_cast <bool> (DVIRec.LocationOps.size() == 1 && "Unexpected number of location ops.") ? void (0) : __assert_fail ("DVIRec.LocationOps.size() == 1 && \"Unexpected number of location ops.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6361, __extension__ __PRETTY_FUNCTION__)) | |||
6361 | "Unexpected number of location ops.")(static_cast <bool> (DVIRec.LocationOps.size() == 1 && "Unexpected number of location ops.") ? void (0) : __assert_fail ("DVIRec.LocationOps.size() == 1 && \"Unexpected number of location ops.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6361, __extension__ __PRETTY_FUNCTION__)); | |||
6362 | // LSR's unsuccessful salvage attempt may have added DIArgList, which in | |||
6363 | // this case was not present before, so force the location back to a single | |||
6364 | // uncontained Value. | |||
6365 | Value *CachedValue = | |||
6366 | getValueOrUndef(DVIRec.LocationOps[0], DVIRec.DVI->getContext()); | |||
6367 | DVIRec.DVI->setRawLocation(ValueAsMetadata::get(CachedValue)); | |||
6368 | } else { | |||
6369 | SmallVector<ValueAsMetadata *, 3> MetadataLocs; | |||
6370 | for (WeakVH VH : DVIRec.LocationOps) { | |||
6371 | Value *CachedValue = getValueOrUndef(VH, DVIRec.DVI->getContext()); | |||
6372 | MetadataLocs.push_back(ValueAsMetadata::get(CachedValue)); | |||
6373 | } | |||
6374 | auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs); | |||
6375 | DVIRec.DVI->setRawLocation( | |||
6376 | llvm::DIArgList::get(DVIRec.DVI->getContext(), ValArrayRef)); | |||
6377 | } | |||
6378 | LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DVIRec.DVI << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: pre-LSR: " << *DVIRec.DVI << '\n'; } } while (false); | |||
6379 | } | |||
6380 | ||||
6381 | static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE, | |||
6382 | llvm::PHINode *LSRInductionVar, DVIRecoveryRec &DVIRec, | |||
6383 | const SCEV *SCEVInductionVar, | |||
6384 | SCEVDbgValueBuilder IterCountExpr) { | |||
6385 | if (!DVIRec.DVI->isKillLocation()) | |||
6386 | return false; | |||
6387 | ||||
6388 | // LSR may have caused several changes to the dbg.value in the failed salvage | |||
6389 | // attempt. So restore the DIExpression, the location ops and also the | |||
6390 | // location ops format, which is always DIArglist for multiple ops, but only | |||
6391 | // sometimes for a single op. | |||
6392 | restorePreTransformState(DVIRec); | |||
6393 | ||||
6394 | // LocationOpIndexMap[i] will store the post-LSR location index of | |||
6395 | // the non-optimised out location at pre-LSR index i. | |||
6396 | SmallVector<int64_t, 2> LocationOpIndexMap; | |||
6397 | LocationOpIndexMap.assign(DVIRec.LocationOps.size(), -1); | |||
6398 | SmallVector<Value *, 2> NewLocationOps; | |||
6399 | NewLocationOps.push_back(LSRInductionVar); | |||
6400 | ||||
6401 | for (unsigned i = 0; i < DVIRec.LocationOps.size(); i++) { | |||
6402 | WeakVH VH = DVIRec.LocationOps[i]; | |||
6403 | // Place the locations not optimised out in the list first, avoiding | |||
6404 | // inserts later. The map is used to update the DIExpression's | |||
6405 | // DW_OP_LLVM_arg arguments as the expression is updated. | |||
6406 | if (VH && !isa<UndefValue>(VH)) { | |||
6407 | NewLocationOps.push_back(VH); | |||
6408 | LocationOpIndexMap[i] = NewLocationOps.size() - 1; | |||
6409 | LLVM_DEBUG(dbgs() << "scev-salvage: Location index " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: Location index " << i << " now at index " << LocationOpIndexMap [i] << "\n"; } } while (false) | |||
6410 | << " now at index " << LocationOpIndexMap[i] << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: Location index " << i << " now at index " << LocationOpIndexMap [i] << "\n"; } } while (false); | |||
6411 | continue; | |||
6412 | } | |||
6413 | ||||
6414 | // It's possible that a value referred to in the SCEV may have been | |||
6415 | // optimised out by LSR. | |||
6416 | if (SE.containsErasedValue(DVIRec.SCEVs[i]) || | |||
6417 | SE.containsUndefs(DVIRec.SCEVs[i])) { | |||
6418 | LLVM_DEBUG(dbgs() << "scev-salvage: SCEV for location at index: " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: SCEV for location at index: " << i << " refers to a location that is now undef or erased. " "Salvage abandoned.\n"; } } while (false) | |||
6419 | << " refers to a location that is now undef or erased. "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: SCEV for location at index: " << i << " refers to a location that is now undef or erased. " "Salvage abandoned.\n"; } } while (false) | |||
6420 | "Salvage abandoned.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: SCEV for location at index: " << i << " refers to a location that is now undef or erased. " "Salvage abandoned.\n"; } } while (false); | |||
6421 | return false; | |||
6422 | } | |||
6423 | ||||
6424 | LLVM_DEBUG(dbgs() << "scev-salvage: salvaging location at index " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: salvaging location at index " << i << " with SCEV: " << *DVIRec.SCEVs[i] << "\n"; } } while (false) | |||
6425 | << " with SCEV: " << *DVIRec.SCEVs[i] << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: salvaging location at index " << i << " with SCEV: " << *DVIRec.SCEVs[i] << "\n"; } } while (false); | |||
6426 | ||||
6427 | DVIRec.RecoveryExprs[i] = std::make_unique<SCEVDbgValueBuilder>(); | |||
6428 | SCEVDbgValueBuilder *SalvageExpr = DVIRec.RecoveryExprs[i].get(); | |||
6429 | ||||
6430 | // Create an offset-based salvage expression if possible, as it requires | |||
6431 | // less DWARF ops than an iteration count-based expression. | |||
6432 | if (std::optional<APInt> Offset = | |||
6433 | SE.computeConstantDifference(DVIRec.SCEVs[i], SCEVInductionVar)) { | |||
6434 | if (Offset->getMinSignedBits() <= 64) | |||
6435 | SalvageExpr->createOffsetExpr(Offset->getSExtValue(), LSRInductionVar); | |||
6436 | } else if (!SalvageExpr->createIterCountExpr(DVIRec.SCEVs[i], IterCountExpr, | |||
6437 | SE)) | |||
6438 | return false; | |||
6439 | } | |||
6440 | ||||
6441 | // Merge the DbgValueBuilder generated expressions and the original | |||
6442 | // DIExpression, place the result into an new vector. | |||
6443 | SmallVector<uint64_t, 3> NewExpr; | |||
6444 | if (DVIRec.Expr->getNumElements() == 0) { | |||
6445 | assert(DVIRec.RecoveryExprs.size() == 1 &&(static_cast <bool> (DVIRec.RecoveryExprs.size() == 1 && "Expected only a single recovery expression for an empty " "DIExpression." ) ? void (0) : __assert_fail ("DVIRec.RecoveryExprs.size() == 1 && \"Expected only a single recovery expression for an empty \" \"DIExpression.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6447, __extension__ __PRETTY_FUNCTION__)) | |||
6446 | "Expected only a single recovery expression for an empty "(static_cast <bool> (DVIRec.RecoveryExprs.size() == 1 && "Expected only a single recovery expression for an empty " "DIExpression." ) ? void (0) : __assert_fail ("DVIRec.RecoveryExprs.size() == 1 && \"Expected only a single recovery expression for an empty \" \"DIExpression.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6447, __extension__ __PRETTY_FUNCTION__)) | |||
6447 | "DIExpression.")(static_cast <bool> (DVIRec.RecoveryExprs.size() == 1 && "Expected only a single recovery expression for an empty " "DIExpression." ) ? void (0) : __assert_fail ("DVIRec.RecoveryExprs.size() == 1 && \"Expected only a single recovery expression for an empty \" \"DIExpression.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6447, __extension__ __PRETTY_FUNCTION__)); | |||
6448 | assert(DVIRec.RecoveryExprs[0] &&(static_cast <bool> (DVIRec.RecoveryExprs[0] && "Expected a SCEVDbgSalvageBuilder for location 0") ? void (0 ) : __assert_fail ("DVIRec.RecoveryExprs[0] && \"Expected a SCEVDbgSalvageBuilder for location 0\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6449, __extension__ __PRETTY_FUNCTION__)) | |||
6449 | "Expected a SCEVDbgSalvageBuilder for location 0")(static_cast <bool> (DVIRec.RecoveryExprs[0] && "Expected a SCEVDbgSalvageBuilder for location 0") ? void (0 ) : __assert_fail ("DVIRec.RecoveryExprs[0] && \"Expected a SCEVDbgSalvageBuilder for location 0\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6449, __extension__ __PRETTY_FUNCTION__)); | |||
6450 | SCEVDbgValueBuilder *B = DVIRec.RecoveryExprs[0].get(); | |||
6451 | B->appendToVectors(NewExpr, NewLocationOps); | |||
6452 | } | |||
6453 | for (const auto &Op : DVIRec.Expr->expr_ops()) { | |||
6454 | // Most Ops needn't be updated. | |||
6455 | if (Op.getOp() != dwarf::DW_OP_LLVM_arg) { | |||
6456 | Op.appendToVector(NewExpr); | |||
6457 | continue; | |||
6458 | } | |||
6459 | ||||
6460 | uint64_t LocationArgIndex = Op.getArg(0); | |||
6461 | SCEVDbgValueBuilder *DbgBuilder = | |||
6462 | DVIRec.RecoveryExprs[LocationArgIndex].get(); | |||
6463 | // The location doesn't have s SCEVDbgValueBuilder, so LSR did not | |||
6464 | // optimise it away. So just translate the argument to the updated | |||
6465 | // location index. | |||
6466 | if (!DbgBuilder) { | |||
6467 | NewExpr.push_back(dwarf::DW_OP_LLVM_arg); | |||
6468 | assert(LocationOpIndexMap[Op.getArg(0)] != -1 &&(static_cast <bool> (LocationOpIndexMap[Op.getArg(0)] != -1 && "Expected a positive index for the location-op position." ) ? void (0) : __assert_fail ("LocationOpIndexMap[Op.getArg(0)] != -1 && \"Expected a positive index for the location-op position.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6469, __extension__ __PRETTY_FUNCTION__)) | |||
6469 | "Expected a positive index for the location-op position.")(static_cast <bool> (LocationOpIndexMap[Op.getArg(0)] != -1 && "Expected a positive index for the location-op position." ) ? void (0) : __assert_fail ("LocationOpIndexMap[Op.getArg(0)] != -1 && \"Expected a positive index for the location-op position.\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6469, __extension__ __PRETTY_FUNCTION__)); | |||
6470 | NewExpr.push_back(LocationOpIndexMap[Op.getArg(0)]); | |||
6471 | continue; | |||
6472 | } | |||
6473 | // The location has a recovery expression. | |||
6474 | DbgBuilder->appendToVectors(NewExpr, NewLocationOps); | |||
6475 | } | |||
6476 | ||||
6477 | UpdateDbgValueInst(DVIRec, NewLocationOps, NewExpr); | |||
6478 | LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: " << *DVIRec.DVI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: Updated DVI: " << *DVIRec.DVI << "\n"; } } while (false); | |||
6479 | return true; | |||
6480 | } | |||
6481 | ||||
6482 | /// Obtain an expression for the iteration count, then attempt to salvage the | |||
6483 | /// dbg.value intrinsics. | |||
6484 | static void | |||
6485 | DbgRewriteSalvageableDVIs(llvm::Loop *L, ScalarEvolution &SE, | |||
6486 | llvm::PHINode *LSRInductionVar, | |||
6487 | SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &DVIToUpdate) { | |||
6488 | if (DVIToUpdate.empty()) | |||
6489 | return; | |||
6490 | ||||
6491 | const llvm::SCEV *SCEVInductionVar = SE.getSCEV(LSRInductionVar); | |||
6492 | assert(SCEVInductionVar &&(static_cast <bool> (SCEVInductionVar && "Anticipated a SCEV for the post-LSR induction variable" ) ? void (0) : __assert_fail ("SCEVInductionVar && \"Anticipated a SCEV for the post-LSR induction variable\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6493, __extension__ __PRETTY_FUNCTION__)) | |||
6493 | "Anticipated a SCEV for the post-LSR induction variable")(static_cast <bool> (SCEVInductionVar && "Anticipated a SCEV for the post-LSR induction variable" ) ? void (0) : __assert_fail ("SCEVInductionVar && \"Anticipated a SCEV for the post-LSR induction variable\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6493, __extension__ __PRETTY_FUNCTION__)); | |||
6494 | ||||
6495 | if (const SCEVAddRecExpr *IVAddRec = | |||
6496 | dyn_cast<SCEVAddRecExpr>(SCEVInductionVar)) { | |||
6497 | if (!IVAddRec->isAffine()) | |||
6498 | return; | |||
6499 | ||||
6500 | // Prevent translation using excessive resources. | |||
6501 | if (IVAddRec->getExpressionSize() > MaxSCEVSalvageExpressionSize) | |||
6502 | return; | |||
6503 | ||||
6504 | // The iteration count is required to recover location values. | |||
6505 | SCEVDbgValueBuilder IterCountExpr; | |||
6506 | IterCountExpr.pushLocation(LSRInductionVar); | |||
6507 | if (!IterCountExpr.SCEVToIterCountExpr(*IVAddRec, SE)) | |||
6508 | return; | |||
6509 | ||||
6510 | LLVM_DEBUG(dbgs() << "scev-salvage: IV SCEV: " << *SCEVInductionVardo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: IV SCEV: " << *SCEVInductionVar << '\n'; } } while (false) | |||
6511 | << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: IV SCEV: " << *SCEVInductionVar << '\n'; } } while (false); | |||
6512 | ||||
6513 | for (auto &DVIRec : DVIToUpdate) { | |||
6514 | SalvageDVI(L, SE, LSRInductionVar, *DVIRec, SCEVInductionVar, | |||
6515 | IterCountExpr); | |||
6516 | } | |||
6517 | } | |||
6518 | } | |||
6519 | ||||
6520 | /// Identify and cache salvageable DVI locations and expressions along with the | |||
6521 | /// corresponding SCEV(s). Also ensure that the DVI is not deleted between | |||
6522 | /// cacheing and salvaging. | |||
6523 | static void DbgGatherSalvagableDVI( | |||
6524 | Loop *L, ScalarEvolution &SE, | |||
6525 | SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &SalvageableDVISCEVs, | |||
6526 | SmallSet<AssertingVH<DbgValueInst>, 2> &DVIHandles) { | |||
6527 | for (const auto &B : L->getBlocks()) { | |||
6528 | for (auto &I : *B) { | |||
6529 | auto DVI = dyn_cast<DbgValueInst>(&I); | |||
6530 | if (!DVI) | |||
6531 | continue; | |||
6532 | // Ensure that if any location op is undef that the dbg.vlue is not | |||
6533 | // cached. | |||
6534 | if (DVI->isKillLocation()) | |||
6535 | continue; | |||
6536 | ||||
6537 | // Check that the location op SCEVs are suitable for translation to | |||
6538 | // DIExpression. | |||
6539 | const auto &HasTranslatableLocationOps = | |||
6540 | [&](const DbgValueInst *DVI) -> bool { | |||
6541 | for (const auto LocOp : DVI->location_ops()) { | |||
6542 | if (!LocOp) | |||
6543 | return false; | |||
6544 | ||||
6545 | if (!SE.isSCEVable(LocOp->getType())) | |||
6546 | return false; | |||
6547 | ||||
6548 | const SCEV *S = SE.getSCEV(LocOp); | |||
6549 | if (SE.containsUndefs(S)) | |||
6550 | return false; | |||
6551 | } | |||
6552 | return true; | |||
6553 | }; | |||
6554 | ||||
6555 | if (!HasTranslatableLocationOps(DVI)) | |||
6556 | continue; | |||
6557 | ||||
6558 | std::unique_ptr<DVIRecoveryRec> NewRec = | |||
6559 | std::make_unique<DVIRecoveryRec>(DVI); | |||
6560 | // Each location Op may need a SCEVDbgValueBuilder in order to recover it. | |||
6561 | // Pre-allocating a vector will enable quick lookups of the builder later | |||
6562 | // during the salvage. | |||
6563 | NewRec->RecoveryExprs.resize(DVI->getNumVariableLocationOps()); | |||
6564 | for (const auto LocOp : DVI->location_ops()) { | |||
6565 | NewRec->SCEVs.push_back(SE.getSCEV(LocOp)); | |||
6566 | NewRec->LocationOps.push_back(LocOp); | |||
6567 | NewRec->HadLocationArgList = DVI->hasArgList(); | |||
6568 | } | |||
6569 | SalvageableDVISCEVs.push_back(std::move(NewRec)); | |||
6570 | DVIHandles.insert(DVI); | |||
6571 | } | |||
6572 | } | |||
6573 | } | |||
6574 | ||||
6575 | /// Ideally pick the PHI IV inserted by ScalarEvolutionExpander. As a fallback | |||
6576 | /// any PHi from the loop header is usable, but may have less chance of | |||
6577 | /// surviving subsequent transforms. | |||
6578 | static llvm::PHINode *GetInductionVariable(const Loop &L, ScalarEvolution &SE, | |||
6579 | const LSRInstance &LSR) { | |||
6580 | ||||
6581 | auto IsSuitableIV = [&](PHINode *P) { | |||
6582 | if (!SE.isSCEVable(P->getType())) | |||
6583 | return false; | |||
6584 | if (const SCEVAddRecExpr *Rec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(P))) | |||
6585 | return Rec->isAffine() && !SE.containsUndefs(SE.getSCEV(P)); | |||
6586 | return false; | |||
6587 | }; | |||
6588 | ||||
6589 | // For now, just pick the first IV that was generated and inserted by | |||
6590 | // ScalarEvolution. Ideally pick an IV that is unlikely to be optimised away | |||
6591 | // by subsequent transforms. | |||
6592 | for (const WeakVH &IV : LSR.getScalarEvolutionIVs()) { | |||
6593 | if (!IV) | |||
6594 | continue; | |||
6595 | ||||
6596 | // There should only be PHI node IVs. | |||
6597 | PHINode *P = cast<PHINode>(&*IV); | |||
6598 | ||||
6599 | if (IsSuitableIV(P)) | |||
6600 | return P; | |||
6601 | } | |||
6602 | ||||
6603 | for (PHINode &P : L.getHeader()->phis()) { | |||
6604 | if (IsSuitableIV(&P)) | |||
6605 | return &P; | |||
6606 | } | |||
6607 | return nullptr; | |||
6608 | } | |||
6609 | ||||
6610 | static std::optional<std::tuple<PHINode *, PHINode *, const SCEV *>> | |||
6611 | canFoldTermCondOfLoop(Loop *L, ScalarEvolution &SE, DominatorTree &DT, | |||
6612 | const LoopInfo &LI) { | |||
6613 | if (!L->isInnermost()) { | |||
6614 | LLVM_DEBUG(dbgs() << "Cannot fold on non-innermost loop\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Cannot fold on non-innermost loop\n" ; } } while (false); | |||
6615 | return std::nullopt; | |||
6616 | } | |||
6617 | // Only inspect on simple loop structure | |||
6618 | if (!L->isLoopSimplifyForm()) { | |||
6619 | LLVM_DEBUG(dbgs() << "Cannot fold on non-simple loop\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Cannot fold on non-simple loop\n" ; } } while (false); | |||
6620 | return std::nullopt; | |||
6621 | } | |||
6622 | ||||
6623 | if (!SE.hasLoopInvariantBackedgeTakenCount(L)) { | |||
6624 | LLVM_DEBUG(dbgs() << "Cannot fold on backedge that is loop variant\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Cannot fold on backedge that is loop variant\n" ; } } while (false); | |||
6625 | return std::nullopt; | |||
6626 | } | |||
6627 | ||||
6628 | BasicBlock *LoopLatch = L->getLoopLatch(); | |||
6629 | ||||
6630 | // TODO: Can we do something for greater than and less than? | |||
6631 | // Terminating condition is foldable when it is an eq/ne icmp | |||
6632 | BranchInst *BI = cast<BranchInst>(LoopLatch->getTerminator()); | |||
6633 | if (BI->isUnconditional()) | |||
6634 | return std::nullopt; | |||
6635 | Value *TermCond = BI->getCondition(); | |||
6636 | if (!isa<ICmpInst>(TermCond) || !cast<ICmpInst>(TermCond)->isEquality()) { | |||
6637 | LLVM_DEBUG(dbgs() << "Cannot fold on branching condition that is not an "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Cannot fold on branching condition that is not an " "ICmpInst::eq / ICmpInst::ne\n"; } } while (false) | |||
6638 | "ICmpInst::eq / ICmpInst::ne\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Cannot fold on branching condition that is not an " "ICmpInst::eq / ICmpInst::ne\n"; } } while (false); | |||
6639 | return std::nullopt; | |||
6640 | } | |||
6641 | if (!TermCond->hasOneUse()) { | |||
6642 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Cannot replace terminating condition with more than one use\n" ; } } while (false) | |||
6643 | dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Cannot replace terminating condition with more than one use\n" ; } } while (false) | |||
6644 | << "Cannot replace terminating condition with more than one use\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Cannot replace terminating condition with more than one use\n" ; } } while (false); | |||
6645 | return std::nullopt; | |||
6646 | } | |||
6647 | ||||
6648 | // For `IsToFold`, a primary IV can be replaced by other affine AddRec when it | |||
6649 | // is only used by the terminating condition. To check for this, we may need | |||
6650 | // to traverse through a chain of use-def until we can examine the final | |||
6651 | // usage. | |||
6652 | // *----------------------* | |||
6653 | // *---->| LoopHeader: | | |||
6654 | // | | PrimaryIV = phi ... | | |||
6655 | // | *----------------------* | |||
6656 | // | | | |||
6657 | // | | | |||
6658 | // | chain of | |||
6659 | // | single use | |||
6660 | // used by | | |||
6661 | // phi | | |||
6662 | // | Value | |||
6663 | // | / \ | |||
6664 | // | chain of chain of | |||
6665 | // | single use single use | |||
6666 | // | / \ | |||
6667 | // | / \ | |||
6668 | // *- Value Value --> used by terminating condition | |||
6669 | auto IsToFold = [&](PHINode &PN) -> bool { | |||
6670 | Value *V = &PN; | |||
6671 | ||||
6672 | while (V->getNumUses() == 1) | |||
6673 | V = *V->user_begin(); | |||
6674 | ||||
6675 | if (V->getNumUses() != 2) | |||
6676 | return false; | |||
6677 | ||||
6678 | Value *VToPN = nullptr; | |||
6679 | Value *VToTermCond = nullptr; | |||
6680 | for (User *U : V->users()) { | |||
6681 | while (U->getNumUses() == 1) { | |||
6682 | if (isa<PHINode>(U)) | |||
6683 | VToPN = U; | |||
6684 | if (U == TermCond) | |||
6685 | VToTermCond = U; | |||
6686 | U = *U->user_begin(); | |||
6687 | } | |||
6688 | } | |||
6689 | return VToPN && VToTermCond; | |||
6690 | }; | |||
6691 | ||||
6692 | // If this is an IV which we could replace the terminating condition, return | |||
6693 | // the final value of the alternative IV on the last iteration. | |||
6694 | auto getAlternateIVEnd = [&](PHINode &PN) -> const SCEV * { | |||
6695 | // FIXME: This does not properly account for overflow. | |||
6696 | const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(SE.getSCEV(&PN)); | |||
6697 | const SCEV *BECount = SE.getBackedgeTakenCount(L); | |||
6698 | const SCEV *TermValueS = SE.getAddExpr( | |||
6699 | AddRec->getOperand(0), | |||
6700 | SE.getTruncateOrZeroExtend( | |||
6701 | SE.getMulExpr( | |||
6702 | AddRec->getOperand(1), | |||
6703 | SE.getTruncateOrZeroExtend( | |||
6704 | SE.getAddExpr(BECount, SE.getOne(BECount->getType())), | |||
6705 | AddRec->getOperand(1)->getType())), | |||
6706 | AddRec->getOperand(0)->getType())); | |||
6707 | const DataLayout &DL = L->getHeader()->getModule()->getDataLayout(); | |||
6708 | SCEVExpander Expander(SE, DL, "lsr_fold_term_cond"); | |||
6709 | if (!Expander.isSafeToExpand(TermValueS)) { | |||
6710 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Is not safe to expand terminating value for phi node" << PN << "\n"; } } while (false) | |||
6711 | dbgs() << "Is not safe to expand terminating value for phi node" << PNdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Is not safe to expand terminating value for phi node" << PN << "\n"; } } while (false) | |||
6712 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Is not safe to expand terminating value for phi node" << PN << "\n"; } } while (false); | |||
6713 | return nullptr; | |||
6714 | } | |||
6715 | return TermValueS; | |||
6716 | }; | |||
6717 | ||||
6718 | PHINode *ToFold = nullptr; | |||
6719 | PHINode *ToHelpFold = nullptr; | |||
6720 | const SCEV *TermValueS = nullptr; | |||
6721 | ||||
6722 | for (PHINode &PN : L->getHeader()->phis()) { | |||
6723 | if (!SE.isSCEVable(PN.getType())) { | |||
6724 | LLVM_DEBUG(dbgs() << "IV of phi '" << PNdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "IV of phi '" << PN << "' is not SCEV-able, not qualified for the " "terminating condition folding.\n" ; } } while (false) | |||
6725 | << "' is not SCEV-able, not qualified for the "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "IV of phi '" << PN << "' is not SCEV-able, not qualified for the " "terminating condition folding.\n" ; } } while (false) | |||
6726 | "terminating condition folding.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "IV of phi '" << PN << "' is not SCEV-able, not qualified for the " "terminating condition folding.\n" ; } } while (false); | |||
6727 | continue; | |||
6728 | } | |||
6729 | const SCEV *S = SE.getSCEV(&PN); | |||
6730 | const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S); | |||
6731 | // Only speculate on affine AddRec | |||
6732 | if (!AddRec || !AddRec->isAffine()) { | |||
6733 | LLVM_DEBUG(dbgs() << "SCEV of phi '" << PNdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "SCEV of phi '" << PN << "' is not an affine add recursion, not qualified " "for the terminating condition folding.\n" ; } } while (false) | |||
6734 | << "' is not an affine add recursion, not qualified "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "SCEV of phi '" << PN << "' is not an affine add recursion, not qualified " "for the terminating condition folding.\n" ; } } while (false) | |||
6735 | "for the terminating condition folding.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "SCEV of phi '" << PN << "' is not an affine add recursion, not qualified " "for the terminating condition folding.\n" ; } } while (false); | |||
6736 | continue; | |||
6737 | } | |||
6738 | ||||
6739 | if (IsToFold(PN)) | |||
6740 | ToFold = &PN; | |||
6741 | else if (auto P = getAlternateIVEnd(PN)) { | |||
6742 | ToHelpFold = &PN; | |||
6743 | TermValueS = P; | |||
6744 | } | |||
6745 | } | |||
6746 | ||||
6747 | LLVM_DEBUG(if (ToFold && !ToHelpFold) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ToFold && !ToHelpFold) dbgs() << "Cannot find other AddRec IV to help folding\n";; } } while ( false) | |||
6748 | << "Cannot find other AddRec IV to help folding\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ToFold && !ToHelpFold) dbgs() << "Cannot find other AddRec IV to help folding\n";; } } while ( false); | |||
6749 | ||||
6750 | LLVM_DEBUG(if (ToFold && ToHelpFold) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ToFold && ToHelpFold) dbgs() << "\nFound loop that can fold terminating condition\n" << " BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n" << " TermCond: " << *TermCond << "\n" << " BrandInst: " << *BI << "\n" << " ToFold: " << *ToFold << "\n" << " ToHelpFold: " << *ToHelpFold << "\n"; } } while (false) | |||
6751 | << "\nFound loop that can fold terminating condition\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ToFold && ToHelpFold) dbgs() << "\nFound loop that can fold terminating condition\n" << " BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n" << " TermCond: " << *TermCond << "\n" << " BrandInst: " << *BI << "\n" << " ToFold: " << *ToFold << "\n" << " ToHelpFold: " << *ToHelpFold << "\n"; } } while (false) | |||
6752 | << " BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ToFold && ToHelpFold) dbgs() << "\nFound loop that can fold terminating condition\n" << " BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n" << " TermCond: " << *TermCond << "\n" << " BrandInst: " << *BI << "\n" << " ToFold: " << *ToFold << "\n" << " ToHelpFold: " << *ToHelpFold << "\n"; } } while (false) | |||
6753 | << " TermCond: " << *TermCond << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ToFold && ToHelpFold) dbgs() << "\nFound loop that can fold terminating condition\n" << " BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n" << " TermCond: " << *TermCond << "\n" << " BrandInst: " << *BI << "\n" << " ToFold: " << *ToFold << "\n" << " ToHelpFold: " << *ToHelpFold << "\n"; } } while (false) | |||
6754 | << " BrandInst: " << *BI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ToFold && ToHelpFold) dbgs() << "\nFound loop that can fold terminating condition\n" << " BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n" << " TermCond: " << *TermCond << "\n" << " BrandInst: " << *BI << "\n" << " ToFold: " << *ToFold << "\n" << " ToHelpFold: " << *ToHelpFold << "\n"; } } while (false) | |||
6755 | << " ToFold: " << *ToFold << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ToFold && ToHelpFold) dbgs() << "\nFound loop that can fold terminating condition\n" << " BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n" << " TermCond: " << *TermCond << "\n" << " BrandInst: " << *BI << "\n" << " ToFold: " << *ToFold << "\n" << " ToHelpFold: " << *ToHelpFold << "\n"; } } while (false) | |||
6756 | << " ToHelpFold: " << *ToHelpFold << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { if (ToFold && ToHelpFold) dbgs() << "\nFound loop that can fold terminating condition\n" << " BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n" << " TermCond: " << *TermCond << "\n" << " BrandInst: " << *BI << "\n" << " ToFold: " << *ToFold << "\n" << " ToHelpFold: " << *ToHelpFold << "\n"; } } while (false); | |||
6757 | ||||
6758 | if (!ToFold || !ToHelpFold) | |||
6759 | return std::nullopt; | |||
6760 | return std::make_tuple(ToFold, ToHelpFold, TermValueS); | |||
6761 | } | |||
6762 | ||||
6763 | static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE, | |||
6764 | DominatorTree &DT, LoopInfo &LI, | |||
6765 | const TargetTransformInfo &TTI, | |||
6766 | AssumptionCache &AC, TargetLibraryInfo &TLI, | |||
6767 | MemorySSA *MSSA) { | |||
6768 | ||||
6769 | // Debug preservation - before we start removing anything identify which DVI | |||
6770 | // meet the salvageable criteria and store their DIExpression and SCEVs. | |||
6771 | SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> SalvageableDVIRecords; | |||
6772 | SmallSet<AssertingVH<DbgValueInst>, 2> DVIHandles; | |||
6773 | DbgGatherSalvagableDVI(L, SE, SalvageableDVIRecords, DVIHandles); | |||
6774 | ||||
6775 | bool Changed = false; | |||
6776 | std::unique_ptr<MemorySSAUpdater> MSSAU; | |||
6777 | if (MSSA) | |||
6778 | MSSAU = std::make_unique<MemorySSAUpdater>(MSSA); | |||
6779 | ||||
6780 | // Run the main LSR transformation. | |||
6781 | const LSRInstance &Reducer = | |||
6782 | LSRInstance(L, IU, SE, DT, LI, TTI, AC, TLI, MSSAU.get()); | |||
6783 | Changed |= Reducer.getChanged(); | |||
6784 | ||||
6785 | // Remove any extra phis created by processing inner loops. | |||
6786 | Changed |= DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get()); | |||
6787 | if (EnablePhiElim && L->isLoopSimplifyForm()) { | |||
6788 | SmallVector<WeakTrackingVH, 16> DeadInsts; | |||
6789 | const DataLayout &DL = L->getHeader()->getModule()->getDataLayout(); | |||
6790 | SCEVExpander Rewriter(SE, DL, "lsr", false); | |||
6791 | #ifndef NDEBUG | |||
6792 | Rewriter.setDebugType(DEBUG_TYPE"loop-reduce"); | |||
6793 | #endif | |||
6794 | unsigned numFolded = Rewriter.replaceCongruentIVs(L, &DT, DeadInsts, &TTI); | |||
6795 | if (numFolded) { | |||
6796 | Changed = true; | |||
6797 | RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts, &TLI, | |||
6798 | MSSAU.get()); | |||
6799 | DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get()); | |||
6800 | } | |||
6801 | } | |||
6802 | // LSR may at times remove all uses of an induction variable from a loop. | |||
6803 | // The only remaining use is the PHI in the exit block. | |||
6804 | // When this is the case, if the exit value of the IV can be calculated using | |||
6805 | // SCEV, we can replace the exit block PHI with the final value of the IV and | |||
6806 | // skip the updates in each loop iteration. | |||
6807 | if (L->isRecursivelyLCSSAForm(DT, LI) && L->getExitBlock()) { | |||
6808 | SmallVector<WeakTrackingVH, 16> DeadInsts; | |||
6809 | const DataLayout &DL = L->getHeader()->getModule()->getDataLayout(); | |||
6810 | SCEVExpander Rewriter(SE, DL, "lsr", true); | |||
6811 | int Rewrites = rewriteLoopExitValues(L, &LI, &TLI, &SE, &TTI, Rewriter, &DT, | |||
6812 | UnusedIndVarInLoop, DeadInsts); | |||
6813 | if (Rewrites) { | |||
6814 | Changed = true; | |||
6815 | RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts, &TLI, | |||
6816 | MSSAU.get()); | |||
6817 | DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get()); | |||
6818 | } | |||
6819 | } | |||
6820 | ||||
6821 | if (AllowTerminatingConditionFoldingAfterLSR) { | |||
6822 | if (auto Opt = canFoldTermCondOfLoop(L, SE, DT, LI)) { | |||
6823 | auto [ToFold, ToHelpFold, TermValueS] = *Opt; | |||
6824 | ||||
6825 | Changed = true; | |||
6826 | NumTermFold++; | |||
6827 | ||||
6828 | BasicBlock *LoopPreheader = L->getLoopPreheader(); | |||
6829 | BasicBlock *LoopLatch = L->getLoopLatch(); | |||
6830 | ||||
6831 | (void)ToFold; | |||
6832 | LLVM_DEBUG(dbgs() << "To fold phi-node:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "To fold phi-node:\n" << *ToFold << "\n" << "New term-cond phi-node:\n" << *ToHelpFold << "\n"; } } while (false) | |||
6833 | << *ToFold << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "To fold phi-node:\n" << *ToFold << "\n" << "New term-cond phi-node:\n" << *ToHelpFold << "\n"; } } while (false) | |||
6834 | << "New term-cond phi-node:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "To fold phi-node:\n" << *ToFold << "\n" << "New term-cond phi-node:\n" << *ToHelpFold << "\n"; } } while (false) | |||
6835 | << *ToHelpFold << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "To fold phi-node:\n" << *ToFold << "\n" << "New term-cond phi-node:\n" << *ToHelpFold << "\n"; } } while (false); | |||
6836 | ||||
6837 | Value *StartValue = ToHelpFold->getIncomingValueForBlock(LoopPreheader); | |||
6838 | (void)StartValue; | |||
6839 | Value *LoopValue = ToHelpFold->getIncomingValueForBlock(LoopLatch); | |||
6840 | ||||
6841 | // SCEVExpander for both use in preheader and latch | |||
6842 | const DataLayout &DL = L->getHeader()->getModule()->getDataLayout(); | |||
6843 | SCEVExpander Expander(SE, DL, "lsr_fold_term_cond"); | |||
6844 | SCEVExpanderCleaner ExpCleaner(Expander); | |||
6845 | ||||
6846 | assert(Expander.isSafeToExpand(TermValueS) &&(static_cast <bool> (Expander.isSafeToExpand(TermValueS ) && "Terminating value was checked safe in canFoldTerminatingCondition" ) ? void (0) : __assert_fail ("Expander.isSafeToExpand(TermValueS) && \"Terminating value was checked safe in canFoldTerminatingCondition\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6847, __extension__ __PRETTY_FUNCTION__)) | |||
6847 | "Terminating value was checked safe in canFoldTerminatingCondition")(static_cast <bool> (Expander.isSafeToExpand(TermValueS ) && "Terminating value was checked safe in canFoldTerminatingCondition" ) ? void (0) : __assert_fail ("Expander.isSafeToExpand(TermValueS) && \"Terminating value was checked safe in canFoldTerminatingCondition\"" , "llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp", 6847, __extension__ __PRETTY_FUNCTION__)); | |||
6848 | ||||
6849 | // Create new terminating value at loop header | |||
6850 | Value *TermValue = Expander.expandCodeFor(TermValueS, ToHelpFold->getType(), | |||
6851 | LoopPreheader->getTerminator()); | |||
6852 | ||||
6853 | LLVM_DEBUG(dbgs() << "Start value of new term-cond phi-node:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Start value of new term-cond phi-node:\n" << *StartValue << "\n" << "Terminating value of new term-cond phi-node:\n" << *TermValue << "\n"; } } while (false) | |||
6854 | << *StartValue << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Start value of new term-cond phi-node:\n" << *StartValue << "\n" << "Terminating value of new term-cond phi-node:\n" << *TermValue << "\n"; } } while (false) | |||
6855 | << "Terminating value of new term-cond phi-node:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Start value of new term-cond phi-node:\n" << *StartValue << "\n" << "Terminating value of new term-cond phi-node:\n" << *TermValue << "\n"; } } while (false) | |||
6856 | << *TermValue << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Start value of new term-cond phi-node:\n" << *StartValue << "\n" << "Terminating value of new term-cond phi-node:\n" << *TermValue << "\n"; } } while (false); | |||
6857 | ||||
6858 | // Create new terminating condition at loop latch | |||
6859 | BranchInst *BI = cast<BranchInst>(LoopLatch->getTerminator()); | |||
6860 | ICmpInst *OldTermCond = cast<ICmpInst>(BI->getCondition()); | |||
6861 | IRBuilder<> LatchBuilder(LoopLatch->getTerminator()); | |||
6862 | // FIXME: We are adding a use of an IV here without account for poison safety. | |||
6863 | // This is incorrect. | |||
6864 | Value *NewTermCond = LatchBuilder.CreateICmp( | |||
6865 | OldTermCond->getPredicate(), LoopValue, TermValue, | |||
6866 | "lsr_fold_term_cond.replaced_term_cond"); | |||
6867 | ||||
6868 | LLVM_DEBUG(dbgs() << "Old term-cond:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Old term-cond:\n" << *OldTermCond << "\n" << "New term-cond:\b" << *NewTermCond << "\n"; } } while (false) | |||
6869 | << *OldTermCond << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Old term-cond:\n" << *OldTermCond << "\n" << "New term-cond:\b" << *NewTermCond << "\n"; } } while (false) | |||
6870 | << "New term-cond:\b" << *NewTermCond << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "Old term-cond:\n" << *OldTermCond << "\n" << "New term-cond:\b" << *NewTermCond << "\n"; } } while (false); | |||
6871 | ||||
6872 | BI->setCondition(NewTermCond); | |||
6873 | ||||
6874 | OldTermCond->eraseFromParent(); | |||
6875 | DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get()); | |||
6876 | ||||
6877 | ExpCleaner.markResultUsed(); | |||
6878 | } | |||
6879 | } | |||
6880 | ||||
6881 | if (SalvageableDVIRecords.empty()) | |||
6882 | return Changed; | |||
6883 | ||||
6884 | // Obtain relevant IVs and attempt to rewrite the salvageable DVIs with | |||
6885 | // expressions composed using the derived iteration count. | |||
6886 | // TODO: Allow for multiple IV references for nested AddRecSCEVs | |||
6887 | for (const auto &L : LI) { | |||
6888 | if (llvm::PHINode *IV = GetInductionVariable(*L, SE, Reducer)) | |||
6889 | DbgRewriteSalvageableDVIs(L, SE, IV, SalvageableDVIRecords); | |||
6890 | else { | |||
6891 | LLVM_DEBUG(dbgs() << "scev-salvage: SCEV salvaging not possible. An IV "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: SCEV salvaging not possible. An IV " "could not be identified.\n"; } } while (false) | |||
6892 | "could not be identified.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("loop-reduce")) { dbgs() << "scev-salvage: SCEV salvaging not possible. An IV " "could not be identified.\n"; } } while (false); | |||
6893 | } | |||
6894 | } | |||
6895 | ||||
6896 | for (auto &Rec : SalvageableDVIRecords) | |||
6897 | Rec->clear(); | |||
6898 | SalvageableDVIRecords.clear(); | |||
6899 | DVIHandles.clear(); | |||
6900 | return Changed; | |||
6901 | } | |||
6902 | ||||
6903 | bool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager & /*LPM*/) { | |||
6904 | if (skipLoop(L)) | |||
6905 | return false; | |||
6906 | ||||
6907 | auto &IU = getAnalysis<IVUsersWrapperPass>().getIU(); | |||
6908 | auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE(); | |||
6909 | auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree(); | |||
6910 | auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); | |||
6911 | const auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI( | |||
6912 | *L->getHeader()->getParent()); | |||
6913 | auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache( | |||
6914 | *L->getHeader()->getParent()); | |||
6915 | auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI( | |||
6916 | *L->getHeader()->getParent()); | |||
6917 | auto *MSSAAnalysis = getAnalysisIfAvailable<MemorySSAWrapperPass>(); | |||
6918 | MemorySSA *MSSA = nullptr; | |||
6919 | if (MSSAAnalysis) | |||
6920 | MSSA = &MSSAAnalysis->getMSSA(); | |||
6921 | return ReduceLoopStrength(L, IU, SE, DT, LI, TTI, AC, TLI, MSSA); | |||
6922 | } | |||
6923 | ||||
6924 | PreservedAnalyses LoopStrengthReducePass::run(Loop &L, LoopAnalysisManager &AM, | |||
6925 | LoopStandardAnalysisResults &AR, | |||
6926 | LPMUpdater &) { | |||
6927 | if (!ReduceLoopStrength(&L, AM.getResult<IVUsersAnalysis>(L, AR), AR.SE, | |||
| ||||
6928 | AR.DT, AR.LI, AR.TTI, AR.AC, AR.TLI, AR.MSSA)) | |||
6929 | return PreservedAnalyses::all(); | |||
6930 | ||||
6931 | auto PA = getLoopPassPreservedAnalyses(); | |||
6932 | if (AR.MSSA) | |||
6933 | PA.preserve<MemorySSAAnalysis>(); | |||
6934 | return PA; | |||
6935 | } | |||
6936 | ||||
6937 | char LoopStrengthReduce::ID = 0; | |||
6938 | ||||
6939 | INITIALIZE_PASS_BEGIN(LoopStrengthReduce, "loop-reduce",static void *initializeLoopStrengthReducePassOnce(PassRegistry &Registry) { | |||
6940 | "Loop Strength Reduction", false, false)static void *initializeLoopStrengthReducePassOnce(PassRegistry &Registry) { | |||
6941 | INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry); | |||
6942 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry); | |||
6943 | INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)initializeScalarEvolutionWrapperPassPass(Registry); | |||
6944 | INITIALIZE_PASS_DEPENDENCY(IVUsersWrapperPass)initializeIVUsersWrapperPassPass(Registry); | |||
6945 | INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)initializeLoopInfoWrapperPassPass(Registry); | |||
6946 | INITIALIZE_PASS_DEPENDENCY(LoopSimplify)initializeLoopSimplifyPass(Registry); | |||
6947 | INITIALIZE_PASS_END(LoopStrengthReduce, "loop-reduce",PassInfo *PI = new PassInfo( "Loop Strength Reduction", "loop-reduce" , &LoopStrengthReduce::ID, PassInfo::NormalCtor_t(callDefaultCtor <LoopStrengthReduce>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeLoopStrengthReducePassFlag ; void llvm::initializeLoopStrengthReducePass(PassRegistry & Registry) { llvm::call_once(InitializeLoopStrengthReducePassFlag , initializeLoopStrengthReducePassOnce, std::ref(Registry)); } | |||
6948 | "Loop Strength Reduction", false, false)PassInfo *PI = new PassInfo( "Loop Strength Reduction", "loop-reduce" , &LoopStrengthReduce::ID, PassInfo::NormalCtor_t(callDefaultCtor <LoopStrengthReduce>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeLoopStrengthReducePassFlag ; void llvm::initializeLoopStrengthReducePass(PassRegistry & Registry) { llvm::call_once(InitializeLoopStrengthReducePassFlag , initializeLoopStrengthReducePassOnce, std::ref(Registry)); } | |||
6949 | ||||
6950 | Pass *llvm::createLoopStrengthReducePass() { return new LoopStrengthReduce(); } |