LLVM 24.0.0git
LoopStrengthReduce.cpp
Go to the documentation of this file.
1//===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This transformation analyzes and transforms the induction variables (and
10// computations derived from them) into forms suitable for efficient execution
11// on the target.
12//
13// This pass performs a strength reduction on array references inside loops that
14// have as one or more of their components the loop induction variable, it
15// rewrites expressions to take advantage of scaled-index addressing modes
16// available on the target, and it performs a variety of other optimizations
17// related to loop induction variables.
18//
19// Terminology note: this code has a lot of handling for "post-increment" or
20// "post-inc" users. This is not talking about post-increment addressing modes;
21// it is instead talking about code like this:
22//
23// %i = phi [ 0, %entry ], [ %i.next, %latch ]
24// ...
25// %i.next = add %i, 1
26// %c = icmp eq %i.next, %n
27//
28// The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however
29// it's useful to think about these as the same register, with some uses using
30// the value of the register before the add and some using it after. In this
31// example, the icmp is a post-increment user, since it uses %i.next, which is
32// the value of the induction variable after the increment. The other common
33// case of post-increment users is users outside the loop.
34//
35// TODO: More sophistication in the way Formulae are generated and filtered.
36//
37// TODO: Handle multiple loops at a time.
38//
39// TODO: Should the addressing mode BaseGV be changed to a ConstantExpr instead
40// of a GlobalValue?
41//
42// TODO: When truncation is free, truncate ICmp users' operands to make it a
43// smaller encoding (on x86 at least).
44//
45// TODO: When a negated register is used by an add (such as in a list of
46// multiple base registers, or as the increment expression in an addrec),
47// we may not actually need both reg and (-1 * reg) in registers; the
48// negation can be implemented by using a sub instead of an add. The
49// lack of support for taking this into consideration when making
50// register pressure decisions is partly worked around by the "Special"
51// use kind.
52//
53//===----------------------------------------------------------------------===//
54
56#include "ScalarOptions.h"
57#include "llvm/ADT/APInt.h"
58#include "llvm/ADT/DenseMap.h"
59#include "llvm/ADT/DenseSet.h"
61#include "llvm/ADT/STLExtras.h"
62#include "llvm/ADT/SetVector.h"
65#include "llvm/ADT/SmallSet.h"
67#include "llvm/ADT/Statistic.h"
85#include "llvm/IR/BasicBlock.h"
86#include "llvm/IR/Constant.h"
87#include "llvm/IR/Constants.h"
90#include "llvm/IR/Dominators.h"
91#include "llvm/IR/GlobalValue.h"
92#include "llvm/IR/IRBuilder.h"
93#include "llvm/IR/InstrTypes.h"
94#include "llvm/IR/Instruction.h"
97#include "llvm/IR/Module.h"
98#include "llvm/IR/Operator.h"
99#include "llvm/IR/Type.h"
100#include "llvm/IR/Use.h"
101#include "llvm/IR/User.h"
102#include "llvm/IR/Value.h"
103#include "llvm/IR/ValueHandle.h"
105#include "llvm/Pass.h"
106#include "llvm/Support/Casting.h"
108#include "llvm/Support/Debug.h"
118#include <algorithm>
119#include <cassert>
120#include <cstddef>
121#include <cstdint>
122#include <iterator>
123#include <limits>
124#include <map>
125#include <numeric>
126#include <optional>
127#include <utility>
128
129using namespace llvm;
130using namespace SCEVPatternMatch;
131
132#define DEBUG_TYPE "loop-reduce"
133
134/// MaxIVUsers is an arbitrary threshold that provides an early opportunity for
135/// bail out. This threshold is far beyond the number of users that LSR can
136/// conceivably solve, so it should not affect generated code, but catches the
137/// worst cases before LSR burns too much compile time and stack space.
138static const unsigned MaxIVUsers = 200;
139
140/// Limit the size of expression that SCEV-based salvaging will attempt to
141/// translate into a DIExpression.
142/// Choose a maximum size such that debuginfo is not excessively increased and
143/// the salvaging is not too expensive for the compiler.
144static const unsigned MaxSCEVSalvageExpressionSize = 64;
145
146static bool stressIVChain(const ScalarOptions &Opts) {
147#ifndef NDEBUG
148 return Opts.stress_ivchain;
149#else
150 return false;
151#endif
152}
153
154namespace {
155
156struct MemAccessTy {
157 /// Used in situations where the accessed memory type is unknown.
158 static const unsigned UnknownAddressSpace =
159 std::numeric_limits<unsigned>::max();
160
161 Type *MemTy = nullptr;
162 unsigned AddrSpace = UnknownAddressSpace;
163
164 MemAccessTy() = default;
165 MemAccessTy(Type *Ty, unsigned AS) : MemTy(Ty), AddrSpace(AS) {}
166
167 bool operator==(MemAccessTy Other) const {
168 return MemTy == Other.MemTy && AddrSpace == Other.AddrSpace;
169 }
170
171 bool operator!=(MemAccessTy Other) const { return !(*this == Other); }
172
173 static MemAccessTy getUnknown(LLVMContext &Ctx,
174 unsigned AS = UnknownAddressSpace) {
175 return MemAccessTy(Type::getVoidTy(Ctx), AS);
176 }
177
178 Type *getType() { return MemTy; }
179};
180
181/// This class holds data which is used to order reuse candidates.
182class RegSortData {
183public:
184 /// This represents the set of LSRUse indices which reference
185 /// a particular register.
186 SmallBitVector UsedByIndices;
187
188 void print(raw_ostream &OS) const;
189 void dump() const;
190};
191
192// An offset from an address that is either scalable or fixed. Used for
193// per-target optimizations of addressing modes.
194class Immediate : public details::FixedOrScalableQuantity<Immediate, int64_t> {
195 constexpr Immediate(ScalarTy MinVal, bool Scalable)
196 : FixedOrScalableQuantity(MinVal, Scalable) {}
197
198 constexpr Immediate(const FixedOrScalableQuantity<Immediate, int64_t> &V)
199 : FixedOrScalableQuantity(V) {}
200
201public:
202 constexpr Immediate() = delete;
203
204 static constexpr Immediate getFixed(ScalarTy MinVal) {
205 return {MinVal, false};
206 }
207 static constexpr Immediate getScalable(ScalarTy MinVal) {
208 return {MinVal, true};
209 }
210 static constexpr Immediate get(ScalarTy MinVal, bool Scalable) {
211 return {MinVal, Scalable};
212 }
213 static constexpr Immediate getZero() { return {0, false}; }
214 static constexpr Immediate getFixedMin() {
215 return {std::numeric_limits<int64_t>::min(), false};
216 }
217 static constexpr Immediate getFixedMax() {
218 return {std::numeric_limits<int64_t>::max(), false};
219 }
220 static constexpr Immediate getScalableMin() {
221 return {std::numeric_limits<int64_t>::min(), true};
222 }
223 static constexpr Immediate getScalableMax() {
224 return {std::numeric_limits<int64_t>::max(), true};
225 }
226
227 constexpr bool isLessThanZero() const { return Quantity < 0; }
228
229 constexpr bool isGreaterThanZero() const { return Quantity > 0; }
230
231 constexpr bool isCompatibleImmediate(const Immediate &Imm) const {
232 return isZero() || Imm.isZero() || Imm.Scalable == Scalable;
233 }
234
235 constexpr bool isMin() const {
236 return Quantity == std::numeric_limits<ScalarTy>::min();
237 }
238
239 constexpr bool isMax() const {
240 return Quantity == std::numeric_limits<ScalarTy>::max();
241 }
242
243 // Arithmetic 'operators' that cast to unsigned types first.
244 constexpr Immediate addUnsigned(const Immediate &RHS) const {
245 assert(isCompatibleImmediate(RHS) && "Incompatible Immediates");
246 ScalarTy Value = (uint64_t)Quantity + RHS.getKnownMinValue();
247 return {Value, Scalable || RHS.isScalable()};
248 }
249
250 constexpr Immediate subUnsigned(const Immediate &RHS) const {
251 assert(isCompatibleImmediate(RHS) && "Incompatible Immediates");
252 ScalarTy Value = (uint64_t)Quantity - RHS.getKnownMinValue();
253 return {Value, Scalable || RHS.isScalable()};
254 }
255
256 // Scale the quantity by a constant without caring about runtime scalability.
257 constexpr Immediate mulUnsigned(const ScalarTy RHS) const {
258 ScalarTy Value = (uint64_t)Quantity * RHS;
259 return {Value, Scalable};
260 }
261
262 // Helpers for generating SCEVs with vscale terms where needed.
263 const SCEV *getSCEV(ScalarEvolution &SE, Type *Ty) const {
264 const SCEV *S = SE.getConstant(Ty, Quantity);
265 if (Scalable)
266 S = SE.getMulExpr(S, SE.getVScale(S->getType()));
267 return S;
268 }
269
270 const SCEV *getNegativeSCEV(ScalarEvolution &SE, Type *Ty) const {
271 const SCEV *NegS = SE.getConstant(Ty, -(uint64_t)Quantity);
272 if (Scalable)
273 NegS = SE.getMulExpr(NegS, SE.getVScale(NegS->getType()));
274 return NegS;
275 }
276
277 const SCEV *getUnknownSCEV(ScalarEvolution &SE, Type *Ty) const {
278 // TODO: Avoid implicit trunc?
279 // See https://github.com/llvm/llvm-project/issues/112510.
280 const SCEV *SU = SE.getUnknown(
281 ConstantInt::getSigned(Ty, Quantity, /*ImplicitTrunc=*/true));
282 if (Scalable)
283 SU = SE.getMulExpr(SU, SE.getVScale(SU->getType()));
284 return SU;
285 }
286};
287
288// This is needed for the Compare type of std::map when Immediate is used
289// as a key. We don't need it to be fully correct against any value of vscale,
290// just to make sure that vscale-related terms in the map are considered against
291// each other rather than being mixed up and potentially missing opportunities.
292struct KeyOrderTargetImmediate {
293 bool operator()(const Immediate &LHS, const Immediate &RHS) const {
294 if (LHS.isScalable() && !RHS.isScalable())
295 return false;
296 if (!LHS.isScalable() && RHS.isScalable())
297 return true;
298 return LHS.getKnownMinValue() < RHS.getKnownMinValue();
299 }
300};
301
302// This would be nicer if we could be generic instead of directly using size_t,
303// but there doesn't seem to be a type trait for is_orderable or
304// is_lessthan_comparable or similar.
305struct KeyOrderSizeTAndImmediate {
306 bool operator()(const std::pair<size_t, Immediate> &LHS,
307 const std::pair<size_t, Immediate> &RHS) const {
308 size_t LSize = LHS.first;
309 size_t RSize = RHS.first;
310 if (LSize != RSize)
311 return LSize < RSize;
312 return KeyOrderTargetImmediate()(LHS.second, RHS.second);
313 }
314};
315} // end anonymous namespace
316
317#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
318void RegSortData::print(raw_ostream &OS) const {
319 OS << "[NumUses=" << UsedByIndices.count() << ']';
320}
321
322LLVM_DUMP_METHOD void RegSortData::dump() const {
323 print(errs()); errs() << '\n';
324}
325#endif
326
327namespace {
328
329/// Map register candidates to information about how they are used.
330class RegUseTracker {
331 using RegUsesTy = DenseMap<const SCEV *, RegSortData>;
332
333 RegUsesTy RegUsesMap;
335
336public:
337 void countRegister(const SCEV *Reg, size_t LUIdx);
338 void dropRegister(const SCEV *Reg, size_t LUIdx);
339 void swapAndDropUse(size_t LUIdx, size_t LastLUIdx);
340
341 bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const;
342
343 const SmallBitVector &getUsedByIndices(const SCEV *Reg) const;
344
345 void clear();
346
349
350 iterator begin() { return RegSequence.begin(); }
351 iterator end() { return RegSequence.end(); }
352 const_iterator begin() const { return RegSequence.begin(); }
353 const_iterator end() const { return RegSequence.end(); }
354};
355
356} // end anonymous namespace
357
358void
359RegUseTracker::countRegister(const SCEV *Reg, size_t LUIdx) {
360 std::pair<RegUsesTy::iterator, bool> Pair = RegUsesMap.try_emplace(Reg);
361 RegSortData &RSD = Pair.first->second;
362 if (Pair.second)
363 RegSequence.push_back(Reg);
364 RSD.UsedByIndices.resize(std::max(RSD.UsedByIndices.size(), LUIdx + 1));
365 RSD.UsedByIndices.set(LUIdx);
366}
367
368void
369RegUseTracker::dropRegister(const SCEV *Reg, size_t LUIdx) {
370 RegUsesTy::iterator It = RegUsesMap.find(Reg);
371 assert(It != RegUsesMap.end());
372 RegSortData &RSD = It->second;
373 assert(RSD.UsedByIndices.size() > LUIdx);
374 RSD.UsedByIndices.reset(LUIdx);
375}
376
377void
378RegUseTracker::swapAndDropUse(size_t LUIdx, size_t LastLUIdx) {
379 assert(LUIdx <= LastLUIdx);
380
381 // Update RegUses. The data structure is not optimized for this purpose;
382 // we must iterate through it and update each of the bit vectors.
383 for (auto &Pair : RegUsesMap) {
384 SmallBitVector &UsedByIndices = Pair.second.UsedByIndices;
385 if (LUIdx < UsedByIndices.size())
386 UsedByIndices[LUIdx] =
387 LastLUIdx < UsedByIndices.size() ? UsedByIndices[LastLUIdx] : false;
388 UsedByIndices.resize(std::min(UsedByIndices.size(), LastLUIdx));
389 }
390}
391
392bool
393RegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const {
394 RegUsesTy::const_iterator I = RegUsesMap.find(Reg);
395 if (I == RegUsesMap.end())
396 return false;
397 const SmallBitVector &UsedByIndices = I->second.UsedByIndices;
398 int i = UsedByIndices.find_first();
399 if (i == -1) return false;
400 if ((size_t)i != LUIdx) return true;
401 return UsedByIndices.find_next(i) != -1;
402}
403
404const SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const {
405 RegUsesTy::const_iterator I = RegUsesMap.find(Reg);
406 assert(I != RegUsesMap.end() && "Unknown register!");
407 return I->second.UsedByIndices;
408}
409
410void RegUseTracker::clear() {
411 RegUsesMap.clear();
412 RegSequence.clear();
413}
414
415namespace {
416
417/// This class holds information that describes a formula for computing
418/// satisfying a use. It may include broken-out immediates and scaled registers.
419struct Formula {
420 /// Global base address used for complex addressing.
421 GlobalValue *BaseGV = nullptr;
422
423 /// Base offset for complex addressing.
424 Immediate BaseOffset = Immediate::getZero();
425
426 /// Whether any complex addressing has a base register.
427 bool HasBaseReg = false;
428
429 /// The scale of any complex addressing.
430 int64_t Scale = 0;
431
432 /// The list of "base" registers for this use. When this is non-empty. The
433 /// canonical representation of a formula is
434 /// 1. BaseRegs.size > 1 implies ScaledReg != NULL and
435 /// 2. ScaledReg != NULL implies Scale != 1 || !BaseRegs.empty().
436 /// 3. The reg containing recurrent expr related with currect loop in the
437 /// formula should be put in the ScaledReg.
438 /// #1 enforces that the scaled register is always used when at least two
439 /// registers are needed by the formula: e.g., reg1 + reg2 is reg1 + 1 * reg2.
440 /// #2 enforces that 1 * reg is reg.
441 /// #3 ensures invariant regs with respect to current loop can be combined
442 /// together in LSR codegen.
443 /// This invariant can be temporarily broken while building a formula.
444 /// However, every formula inserted into the LSRInstance must be in canonical
445 /// form.
447
448 /// The 'scaled' register for this use. This should be non-null when Scale is
449 /// not zero.
450 const SCEV *ScaledReg = nullptr;
451
452 /// An additional constant offset which added near the use. This requires a
453 /// temporary register, but the offset itself can live in an add immediate
454 /// field rather than a register.
455 Immediate UnfoldedOffset = Immediate::getZero();
456
457 Formula() = default;
458
459 void initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE);
460
461 bool isCanonical(const Loop &L) const;
462
463 void canonicalize(const Loop &L);
464
465 bool unscale();
466
467 bool hasZeroEnd() const;
468
469 bool countsDownToZero() const;
470
471 size_t getNumRegs() const;
472 Type *getType() const;
473
474 void deleteBaseReg(const SCEV *&S);
475
476 bool referencesReg(const SCEV *S) const;
477 bool hasRegsUsedByUsesOtherThan(size_t LUIdx,
478 const RegUseTracker &RegUses) const;
479
480 void print(raw_ostream &OS) const;
481 void dump() const;
482};
483
484} // end anonymous namespace
485
486/// Recursion helper for initialMatch.
487static void DoInitialMatch(const SCEV *S, Loop *L,
490 // Collect expressions which properly dominate the loop header.
491 if (SE.properlyDominates(S, L->getHeader())) {
492 Good.push_back(S);
493 return;
494 }
495
496 // Look at add operands.
497 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
498 for (const SCEV *S : Add->operands())
499 DoInitialMatch(S, L, Good, Bad, SE);
500 return;
501 }
502
503 // Look at addrec operands.
504 const SCEV *Start, *Step;
505 const Loop *ARLoop;
506 if (match(S,
507 m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step), m_Loop(ARLoop))) &&
508 !Start->isZero()) {
509 DoInitialMatch(Start, L, Good, Bad, SE);
510 DoInitialMatch(SE.getAddRecExpr(SE.getConstant(S->getType(), 0), Step,
511 // FIXME: AR->getNoWrapFlags()
512 ARLoop, SCEV::FlagNone),
513 L, Good, Bad, SE);
514 return;
515 }
516
517 // Handle a multiplication by -1 (negation) if it didn't fold.
518 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S))
519 if (Mul->getOperand(0)->isAllOnesValue()) {
521 const SCEV *NewMul = SE.getMulExpr(Ops);
522
525 DoInitialMatch(NewMul, L, MyGood, MyBad, SE);
526 const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue(
527 SE.getEffectiveSCEVType(NewMul->getType())));
528 for (const SCEV *S : MyGood)
529 Good.push_back(SE.getMulExpr(NegOne, S));
530 for (const SCEV *S : MyBad)
531 Bad.push_back(SE.getMulExpr(NegOne, S));
532 return;
533 }
534
535 // Ok, we can't do anything interesting. Just stuff the whole thing into a
536 // register and hope for the best.
537 Bad.push_back(S);
538}
539
540/// Incorporate loop-variant parts of S into this Formula, attempting to keep
541/// all loop-invariant and loop-computable values in a single base register.
542void Formula::initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) {
545 DoInitialMatch(S, L, Good, Bad, SE);
546 if (!Good.empty()) {
547 const SCEV *Sum = SE.getAddExpr(Good);
548 if (!Sum->isZero())
549 BaseRegs.push_back(Sum);
550 HasBaseReg = true;
551 }
552 if (!Bad.empty()) {
553 const SCEV *Sum = SE.getAddExpr(Bad);
554 if (!Sum->isZero())
555 BaseRegs.push_back(Sum);
556 HasBaseReg = true;
557 }
558 canonicalize(*L);
559}
560
561static bool containsAddRecDependentOnLoop(const SCEV *S, const Loop &L) {
562 return SCEVExprContains(S, [&L](const SCEV *S) {
563 return isa<SCEVAddRecExpr>(S) && (cast<SCEVAddRecExpr>(S)->getLoop() == &L);
564 });
565}
566
567/// Check whether or not this formula satisfies the canonical
568/// representation.
569/// \see Formula::BaseRegs.
570bool Formula::isCanonical(const Loop &L) const {
571 assert((Scale == 0 || ScaledReg) &&
572 "ScaledReg must be non-null if Scale is non-zero");
573
574 if (!ScaledReg)
575 return BaseRegs.size() <= 1;
576
577 if (Scale != 1)
578 return true;
579
580 if (Scale == 1 && BaseRegs.empty())
581 return false;
582
583 if (containsAddRecDependentOnLoop(ScaledReg, L))
584 return true;
585
586 // If ScaledReg is not a recurrent expr, or it is but its loop is not current
587 // loop, meanwhile BaseRegs contains a recurrent expr reg related with current
588 // loop, we want to swap the reg in BaseRegs with ScaledReg.
589 return none_of(BaseRegs, [&L](const SCEV *S) {
591 });
592}
593
594/// Helper method to morph a formula into its canonical representation.
595/// \see Formula::BaseRegs.
596/// Every formula having more than one base register, must use the ScaledReg
597/// field. Otherwise, we would have to do special cases everywhere in LSR
598/// to treat reg1 + reg2 + ... the same way as reg1 + 1*reg2 + ...
599/// On the other hand, 1*reg should be canonicalized into reg.
600void Formula::canonicalize(const Loop &L) {
601 if (isCanonical(L))
602 return;
603
604 if (BaseRegs.empty()) {
605 // No base reg? Use scale reg with scale = 1 as such.
606 assert(ScaledReg && "Expected 1*reg => reg");
607 assert(Scale == 1 && "Expected 1*reg => reg");
608 BaseRegs.push_back(ScaledReg);
609 Scale = 0;
610 ScaledReg = nullptr;
611 return;
612 }
613
614 // Keep the invariant sum in BaseRegs and one of the variant sum in ScaledReg.
615 if (!ScaledReg) {
616 ScaledReg = BaseRegs.pop_back_val();
617 Scale = 1;
618 }
619
620 // If ScaledReg is an invariant with respect to L, find the reg from
621 // BaseRegs containing the recurrent expr related with Loop L. Swap the
622 // reg with ScaledReg.
623 if (!containsAddRecDependentOnLoop(ScaledReg, L)) {
624 auto I = find_if(BaseRegs, [&L](const SCEV *S) {
626 });
627 if (I != BaseRegs.end())
628 std::swap(ScaledReg, *I);
629 }
630 assert(isCanonical(L) && "Failed to canonicalize?");
631}
632
633/// Get rid of the scale in the formula.
634/// In other words, this method morphes reg1 + 1*reg2 into reg1 + reg2.
635/// \return true if it was possible to get rid of the scale, false otherwise.
636/// \note After this operation the formula may not be in the canonical form.
637bool Formula::unscale() {
638 if (Scale != 1)
639 return false;
640 Scale = 0;
641 BaseRegs.push_back(ScaledReg);
642 ScaledReg = nullptr;
643 return true;
644}
645
646bool Formula::hasZeroEnd() const {
647 if (UnfoldedOffset || BaseOffset)
648 return false;
649 if (BaseRegs.size() != 1 || ScaledReg)
650 return false;
651 return true;
652}
653
654bool Formula::countsDownToZero() const {
655 if (!hasZeroEnd())
656 return false;
657 assert(BaseRegs.size() == 1 && "hasZeroEnd should mean one BaseReg");
658 const APInt *StepInt;
659 if (!match(BaseRegs[0], m_scev_AffineAddRec(m_SCEV(), m_scev_APInt(StepInt))))
660 return false;
661 return StepInt->isNegative();
662}
663
664/// Return the total number of register operands used by this formula. This does
665/// not include register uses implied by non-constant addrec strides.
666size_t Formula::getNumRegs() const {
667 return !!ScaledReg + BaseRegs.size();
668}
669
670/// Return the type of this formula, if it has one, or null otherwise. This type
671/// is meaningless except for the bit size.
672Type *Formula::getType() const {
673 return !BaseRegs.empty() ? BaseRegs.front()->getType() :
674 ScaledReg ? ScaledReg->getType() :
675 BaseGV ? BaseGV->getType() :
676 nullptr;
677}
678
679/// Delete the given base reg from the BaseRegs list.
680void Formula::deleteBaseReg(const SCEV *&S) {
681 if (&S != &BaseRegs.back())
682 std::swap(S, BaseRegs.back());
683 BaseRegs.pop_back();
684}
685
686/// Test if this formula references the given register.
687bool Formula::referencesReg(const SCEV *S) const {
688 return S == ScaledReg || is_contained(BaseRegs, S);
689}
690
691/// Test whether this formula uses registers which are used by uses other than
692/// the use with the given index.
693bool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx,
694 const RegUseTracker &RegUses) const {
695 if (ScaledReg)
696 if (RegUses.isRegUsedByUsesOtherThan(ScaledReg, LUIdx))
697 return true;
698 for (const SCEV *BaseReg : BaseRegs)
699 if (RegUses.isRegUsedByUsesOtherThan(BaseReg, LUIdx))
700 return true;
701 return false;
702}
703
704#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
705void Formula::print(raw_ostream &OS) const {
706 ListSeparator Plus(" + ");
707 if (BaseGV) {
708 OS << Plus;
709 BaseGV->printAsOperand(OS, /*PrintType=*/false);
710 }
711 if (BaseOffset.isNonZero())
712 OS << Plus << BaseOffset;
713
714 for (const SCEV *BaseReg : BaseRegs)
715 OS << Plus << "reg(" << *BaseReg << ')';
716
717 if (HasBaseReg && BaseRegs.empty())
718 OS << Plus << "**error: HasBaseReg**";
719 else if (!HasBaseReg && !BaseRegs.empty())
720 OS << Plus << "**error: !HasBaseReg**";
721
722 if (Scale != 0) {
723 OS << Plus << Scale << "*reg(";
724 if (ScaledReg)
725 OS << *ScaledReg;
726 else
727 OS << "<unknown>";
728 OS << ')';
729 }
730 if (UnfoldedOffset.isNonZero())
731 OS << Plus << "imm(" << UnfoldedOffset << ')';
732}
733
734LLVM_DUMP_METHOD void Formula::dump() const {
735 print(errs()); errs() << '\n';
736}
737#endif
738
739/// Return true if the given addrec can be sign-extended without changing its
740/// value.
742 Type *WideTy =
744 return isa<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy));
745}
746
747/// Return true if the given add can be sign-extended without changing its
748/// value.
749static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) {
750 Type *WideTy =
751 IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(A->getType()) + 1);
752 return isa<SCEVAddExpr>(SE.getSignExtendExpr(A, WideTy));
753}
754
755/// Return true if the given mul can be sign-extended without changing its
756/// value.
757static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) {
758 Type *WideTy =
760 SE.getTypeSizeInBits(M->getType()) * M->getNumOperands());
761 return isa<SCEVMulExpr>(SE.getSignExtendExpr(M, WideTy));
762}
763
764/// Return an expression for LHS /s RHS, if it can be determined and if the
765/// remainder is known to be zero, or null otherwise. If IgnoreSignificantBits
766/// is true, expressions like (X * Y) /s Y are simplified to X, ignoring that
767/// the multiplication may overflow, which is useful when the result will be
768/// used in a context where the most significant bits are ignored.
769static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
770 ScalarEvolution &SE,
771 bool IgnoreSignificantBits = false) {
772 // A stride can fold to zero once the callers sign-extend it.
773 if (RHS->isZero())
774 return nullptr;
775
776 // Handle the trivial case, which works for any SCEV type.
777 if (LHS == RHS)
778 return SE.getConstant(LHS->getType(), 1);
779
780 // Handle a few RHS special cases.
782 if (RC) {
783 const APInt &RA = RC->getAPInt();
784 // Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do
785 // some folding.
786 if (RA.isAllOnes()) {
787 if (LHS->getType()->isPointerTy())
788 return nullptr;
789 return SE.getMulExpr(LHS, RC);
790 }
791 // Handle x /s 1 as x.
792 if (RA == 1)
793 return LHS;
794 }
795
796 // Check for a division of a constant by a constant.
798 if (!RC)
799 return nullptr;
800 const APInt &LA = C->getAPInt();
801 const APInt &RA = RC->getAPInt();
802 if (LA.srem(RA) != 0)
803 return nullptr;
804 return SE.getConstant(LA.sdiv(RA));
805 }
806
807 // Distribute the sdiv over addrec operands, if the addrec doesn't overflow.
809 if ((IgnoreSignificantBits || isAddRecSExtable(AR, SE)) && AR->isAffine()) {
810 const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE,
811 IgnoreSignificantBits);
812 if (!Step) return nullptr;
813 const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE,
814 IgnoreSignificantBits);
815 if (!Start) return nullptr;
816 // FlagNW is independent of the start value, step direction, and is
817 // preserved with smaller magnitude steps.
818 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
819 return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagNone);
820 }
821 return nullptr;
822 }
823
824 // Distribute the sdiv over add operands, if the add doesn't overflow.
826 if (IgnoreSignificantBits || isAddSExtable(Add, SE)) {
828 for (const SCEV *S : Add->operands()) {
829 const SCEV *Op = getExactSDiv(S, RHS, SE, IgnoreSignificantBits);
830 if (!Op) return nullptr;
831 Ops.push_back(Op);
832 }
833 return SE.getAddExpr(Ops);
834 }
835 return nullptr;
836 }
837
838 // Check for a multiply operand that we can pull RHS out of.
840 if (IgnoreSignificantBits || isMulSExtable(Mul, SE)) {
841 // Handle special case C1*X*Y /s C2*X*Y.
842 if (const SCEVMulExpr *MulRHS = dyn_cast<SCEVMulExpr>(RHS)) {
843 if (IgnoreSignificantBits || isMulSExtable(MulRHS, SE)) {
844 const SCEVConstant *LC = dyn_cast<SCEVConstant>(Mul->getOperand(0));
845 const SCEVConstant *RC =
846 dyn_cast<SCEVConstant>(MulRHS->getOperand(0));
847 if (LC && RC) {
849 SmallVector<const SCEV *, 4> ROps(drop_begin(MulRHS->operands()));
850 if (LOps == ROps)
851 return getExactSDiv(LC, RC, SE, IgnoreSignificantBits);
852 }
853 }
854 }
855
857 bool Found = false;
858 for (const SCEV *S : Mul->operands()) {
859 if (!Found)
860 if (const SCEV *Q = getExactSDiv(S, RHS, SE,
861 IgnoreSignificantBits)) {
862 S = Q;
863 Found = true;
864 }
865 Ops.push_back(S);
866 }
867 return Found ? SE.getMulExpr(Ops) : nullptr;
868 }
869 return nullptr;
870 }
871
872 // Otherwise we don't know.
873 return nullptr;
874}
875
876/// Extracts an immediate operand from \p Ops and replaces the operand with
877/// zero. If \p PreferScalable is true and \p Ops contains both a scalable and
878/// non-scalable offsets, the scalable offset will be extracted.
879static Immediate extractImmediateOperand(const ScalarOptions &Opts,
881 ScalarEvolution &SE,
882 bool PreferScalable) {
883 const APInt *C;
884 SCEVUse *Op = nullptr;
885 Immediate Result = Immediate::getZero();
886
887 // Ops are sorted by their SCEVType (the order of SCEVTypes enum). So, for an
888 // AddExpr the possible order of operands is:
889 // Constant < VScale < Truncate < ZeroExtend < SignExtend < MulExpr < ...
890
891 // This means fixed-size immediates will always appear on the LHS:
892 SCEVUse &S = Ops.front();
893 if (match(S, m_scev_APInt(C)) && !C->isZero() &&
894 C->getSignificantBits() <= 64) {
895 Op = &S;
896 Result = Immediate::getFixed(C->getSExtValue());
897 }
898
899 // But scalable immediates, which are MulExpr(Vscale, Constant), can appear
900 // later in the operand list:
901 if (Opts.lsr_enable_vscale_immediates &&
902 (Result.isZero() || PreferScalable)) {
903 for (SCEVUse &S : Ops) {
904 // We know anything past scMulExpr will not be a vscale immediate.
905 if (S->getSCEVType() > scMulExpr)
906 break;
908 Op = &S;
909 Result = Immediate::getScalable(C->getSExtValue());
910 break;
911 }
912 }
913 }
914
915 if (Result.isNonZero()) {
916 SCEVUse &S = *Op;
917 S = SE.getConstant(S->getType(), 0);
918 }
919
920 return Result;
921}
922
923/// If S involves the addition of a constant integer value, return that integer
924/// value, and mutate S to point to a new SCEV with that value excluded.
925static Immediate extractImmediate(const ScalarOptions &Opts, SCEVUse &S,
926 ScalarEvolution &SE,
927 bool PreferScalable = false) {
928 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
929 SmallVector<SCEVUse, 8> NewOps(Add->operands());
930 Immediate Result =
931 extractImmediateOperand(Opts, NewOps, SE, PreferScalable);
932 if (Result.isZero())
933 Result = extractImmediate(Opts, NewOps.front(), SE, PreferScalable);
934 if (Result.isNonZero())
935 S = SE.getAddExpr(NewOps);
936 return Result;
937 } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
938 SmallVector<SCEVUse, 8> NewOps(AR->operands());
939 Immediate Result =
940 extractImmediate(Opts, NewOps.front(), SE, PreferScalable);
941 if (Result.isNonZero())
942 S = SE.getAddRecExpr(NewOps, AR->getLoop(),
943 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
945 return Result;
946 }
947 return extractImmediateOperand(Opts, {S}, SE, PreferScalable);
948}
949
950/// If S involves the addition of a GlobalValue address, return that symbol, and
951/// mutate S to point to a new SCEV with that value excluded.
953 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
954 if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) {
955 S = SE.getConstant(GV->getType(), 0);
956 return GV;
957 }
958 } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
959 SmallVector<SCEVUse, 8> NewOps(Add->operands());
960 GlobalValue *Result = ExtractSymbol(NewOps.back(), SE);
961 if (Result)
962 S = SE.getAddExpr(NewOps);
963 return Result;
964 } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
965 SmallVector<SCEVUse, 8> NewOps(AR->operands());
966 GlobalValue *Result = ExtractSymbol(NewOps.front(), SE);
967 if (Result)
968 S = SE.getAddRecExpr(NewOps, AR->getLoop(),
969 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
971 return Result;
972 }
973 return nullptr;
974}
975
976/// Returns true if the specified instruction is using the specified value as an
977/// address.
979 Instruction *Inst, Value *OperandVal) {
980 bool isAddress = isa<LoadInst>(Inst);
981 if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
982 if (SI->getPointerOperand() == OperandVal)
983 isAddress = true;
984 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
985 // Addressing modes can also be folded into prefetches and a variety
986 // of intrinsics.
987 switch (II->getIntrinsicID()) {
988 case Intrinsic::memset:
989 case Intrinsic::prefetch:
990 case Intrinsic::masked_load:
991 if (II->getArgOperand(0) == OperandVal)
992 isAddress = true;
993 break;
994 case Intrinsic::masked_store:
995 if (II->getArgOperand(1) == OperandVal)
996 isAddress = true;
997 break;
998 case Intrinsic::memmove:
999 case Intrinsic::memcpy:
1000 if (II->getArgOperand(0) == OperandVal ||
1001 II->getArgOperand(1) == OperandVal)
1002 isAddress = true;
1003 break;
1004 default: {
1005 MemIntrinsicInfo IntrInfo;
1006 if (TTI.getTgtMemIntrinsic(II, IntrInfo)) {
1007 if (IntrInfo.PtrVal == OperandVal)
1008 isAddress = true;
1009 }
1010 }
1011 }
1012 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Inst)) {
1013 if (RMW->getPointerOperand() == OperandVal)
1014 isAddress = true;
1015 } else if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Inst)) {
1016 if (CmpX->getPointerOperand() == OperandVal)
1017 isAddress = true;
1018 }
1019 return isAddress;
1020}
1021
1022/// Return the type of the memory being accessed.
1023static MemAccessTy getAccessType(const TargetTransformInfo &TTI,
1024 Instruction *Inst, Value *OperandVal) {
1025 MemAccessTy AccessTy = MemAccessTy::getUnknown(Inst->getContext());
1026
1027 // First get the type of memory being accessed.
1028 if (Type *Ty = Inst->getAccessType())
1029 AccessTy.MemTy = Ty;
1030
1031 // Then get the pointer address space.
1032 if (const StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
1033 AccessTy.AddrSpace = SI->getPointerAddressSpace();
1034 } else if (const LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
1035 AccessTy.AddrSpace = LI->getPointerAddressSpace();
1036 } else if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Inst)) {
1037 AccessTy.AddrSpace = RMW->getPointerAddressSpace();
1038 } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Inst)) {
1039 AccessTy.AddrSpace = CmpX->getPointerAddressSpace();
1040 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
1041 switch (II->getIntrinsicID()) {
1042 case Intrinsic::prefetch:
1043 case Intrinsic::memset:
1044 AccessTy.AddrSpace = II->getArgOperand(0)->getType()->getPointerAddressSpace();
1045 AccessTy.MemTy = OperandVal->getType();
1046 break;
1047 case Intrinsic::memmove:
1048 case Intrinsic::memcpy:
1049 AccessTy.AddrSpace = OperandVal->getType()->getPointerAddressSpace();
1050 AccessTy.MemTy = OperandVal->getType();
1051 break;
1052 case Intrinsic::masked_load:
1053 AccessTy.AddrSpace =
1054 II->getArgOperand(0)->getType()->getPointerAddressSpace();
1055 break;
1056 case Intrinsic::masked_store:
1057 AccessTy.AddrSpace =
1058 II->getArgOperand(1)->getType()->getPointerAddressSpace();
1059 break;
1060 default: {
1061 MemIntrinsicInfo IntrInfo;
1062 if (TTI.getTgtMemIntrinsic(II, IntrInfo) && IntrInfo.PtrVal) {
1063 AccessTy.AddrSpace
1064 = IntrInfo.PtrVal->getType()->getPointerAddressSpace();
1065 }
1066
1067 break;
1068 }
1069 }
1070 }
1071
1072 return AccessTy;
1073}
1074
1075/// Return true if this AddRec is already a phi in its loop.
1076static bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
1077 for (PHINode &PN : AR->getLoop()->getHeader()->phis()) {
1078 if (SE.isSCEVable(PN.getType()) &&
1079 (SE.getEffectiveSCEVType(PN.getType()) ==
1080 SE.getEffectiveSCEVType(AR->getType())) &&
1081 SE.getSCEV(&PN) == AR)
1082 return true;
1083 }
1084 return false;
1085}
1086
1087/// Check if expanding this expression is likely to incur significant cost. This
1088/// is tricky because SCEV doesn't track which expressions are actually computed
1089/// by the current IR.
1090///
1091/// We currently allow expansion of IV increments that involve adds,
1092/// multiplication by constants, and AddRecs from existing phis.
1093///
1094/// TODO: Allow UDivExpr if we can find an existing IV increment that is an
1095/// obvious multiple of the UDivExpr.
1096static bool isHighCostExpansion(const SCEV *S,
1098 ScalarEvolution &SE) {
1099 // Zero/One operand expressions
1100 switch (S->getSCEVType()) {
1101 case scUnknown:
1102 case scConstant:
1103 case scVScale:
1104 return false;
1105 case scTruncate:
1106 return isHighCostExpansion(cast<SCEVTruncateExpr>(S)->getOperand(),
1107 Processed, SE);
1108 case scZeroExtend:
1109 return isHighCostExpansion(cast<SCEVZeroExtendExpr>(S)->getOperand(),
1110 Processed, SE);
1111 case scSignExtend:
1112 return isHighCostExpansion(cast<SCEVSignExtendExpr>(S)->getOperand(),
1113 Processed, SE);
1114 default:
1115 break;
1116 }
1117
1118 if (!Processed.insert(S).second)
1119 return false;
1120
1121 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
1122 for (const SCEV *S : Add->operands()) {
1123 if (isHighCostExpansion(S, Processed, SE))
1124 return true;
1125 }
1126 return false;
1127 }
1128
1129 const SCEV *Op0, *Op1;
1130 if (match(S, m_scev_Mul(m_SCEV(Op0), m_SCEV(Op1)))) {
1131 // Multiplication by a constant is ok
1132 if (isa<SCEVConstant>(Op0))
1133 return isHighCostExpansion(Op1, Processed, SE);
1134
1135 // If we have the value of one operand, check if an existing
1136 // multiplication already generates this expression.
1137 if (const auto *U = dyn_cast<SCEVUnknown>(Op1)) {
1138 Value *UVal = U->getValue();
1139 for (User *UR : UVal->users()) {
1140 // If U is a constant, it may be used by a ConstantExpr.
1142 if (UI && UI->getOpcode() == Instruction::Mul &&
1143 SE.isSCEVable(UI->getType())) {
1144 return SE.getSCEV(UI) == S;
1145 }
1146 }
1147 }
1148 }
1149
1150 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
1151 if (isExistingPhi(AR, SE))
1152 return false;
1153 }
1154
1155 // Fow now, consider any other type of expression (div/mul/min/max) high cost.
1156 return true;
1157}
1158
1159namespace {
1160
1161class LSRUse;
1162
1163} // end anonymous namespace
1164
1165/// Check if the addressing mode defined by \p F is completely
1166/// folded in \p LU at isel time.
1167/// This includes address-mode folding and special icmp tricks.
1168/// This function returns true if \p LU can accommodate what \p F
1169/// defines and up to 1 base + 1 scaled + offset.
1170/// In other words, if \p F has several base registers, this function may
1171/// still return true. Therefore, users still need to account for
1172/// additional base registers and/or unfolded offsets to derive an
1173/// accurate cost model.
1174static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1175 const LSRUse &LU, const Formula &F);
1176
1177// Get the cost of the scaling factor used in F for LU.
1178static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI,
1179 const LSRUse &LU, const Formula &F,
1180 const Loop &L);
1181
1182namespace {
1183
1184/// This class is used to measure and compare candidate formulae.
1185class Cost {
1186 const ScalarOptions *Opts = nullptr;
1187 const Loop *L = nullptr;
1188 ScalarEvolution *SE = nullptr;
1189 const TargetTransformInfo *TTI = nullptr;
1190 TargetTransformInfo::LSRCost C;
1192
1193public:
1194 Cost() = delete;
1195 Cost(const ScalarOptions &Opts, const Loop *L, ScalarEvolution &SE,
1196 const TargetTransformInfo &TTI, TTI::AddressingModeKind AMK)
1197 : Opts(&Opts), L(L), SE(&SE), TTI(&TTI), AMK(AMK) {
1198 C.Insns = 0;
1199 C.NumRegs = 0;
1200 C.AddRecCost = 0;
1201 C.NumIVMuls = 0;
1202 C.NumBaseAdds = 0;
1203 C.ImmCost = 0;
1204 C.SetupCost = 0;
1205 C.ScaleCost = 0;
1206 C.NumPrePostIndex = 0;
1207 }
1208
1209 bool isLess(const Cost &Other) const;
1210
1211 void Lose();
1212
1213#ifndef NDEBUG
1214 // Once any of the metrics loses, they must all remain losers.
1215 bool isValid() {
1216 return ((C.Insns | C.NumRegs | C.AddRecCost | C.NumIVMuls | C.NumBaseAdds |
1217 C.ImmCost | C.SetupCost | C.ScaleCost | C.NumPrePostIndex) !=
1218 ~0u) ||
1219 ((C.Insns & C.NumRegs & C.AddRecCost & C.NumIVMuls & C.NumBaseAdds &
1220 C.ImmCost & C.SetupCost & C.ScaleCost & C.NumPrePostIndex) == ~0u);
1221 }
1222#endif
1223
1224 bool isLoser() {
1225 assert(isValid() && "invalid cost");
1226 return C.NumRegs == ~0u;
1227 }
1228
1229 void RateFormula(const Formula &F, SmallPtrSetImpl<const SCEV *> &Regs,
1230 const DenseSet<const SCEV *> &VisitedRegs, const LSRUse &LU,
1231 bool HardwareLoopProfitable,
1232 SmallPtrSetImpl<const SCEV *> *LoserRegs = nullptr);
1233
1234 void print(raw_ostream &OS) const;
1235 void dump() const;
1236
1237private:
1238 void RateRegister(const Formula &F, const SCEV *Reg,
1239 SmallPtrSetImpl<const SCEV *> &Regs, const LSRUse &LU,
1240 bool HardwareLoopProfitable);
1241 void RatePrimaryRegister(const Formula &F, const SCEV *Reg,
1242 SmallPtrSetImpl<const SCEV *> &Regs,
1243 const LSRUse &LU, bool HardwareLoopProfitable,
1244 SmallPtrSetImpl<const SCEV *> *LoserRegs);
1245};
1246
1247/// An operand value in an instruction which is to be replaced with some
1248/// equivalent, possibly strength-reduced, replacement.
1249struct LSRFixup {
1250 /// The instruction which will be updated.
1251 Instruction *UserInst = nullptr;
1252
1253 /// The operand of the instruction which will be replaced. The operand may be
1254 /// used more than once; every instance will be replaced.
1255 Value *OperandValToReplace = nullptr;
1256
1257 /// If this user is to use the post-incremented value of an induction
1258 /// variable, this set is non-empty and holds the loops associated with the
1259 /// induction variable.
1260 PostIncLoopSet PostIncLoops;
1261
1262 /// A constant offset to be added to the LSRUse expression. This allows
1263 /// multiple fixups to share the same LSRUse with different offsets, for
1264 /// example in an unrolled loop.
1265 Immediate Offset = Immediate::getZero();
1266
1267 LSRFixup() = default;
1268
1269 bool isUseFullyOutsideLoop(const Loop *L) const;
1270
1271 void print(raw_ostream &OS) const;
1272 void dump() const;
1273};
1274
1275/// This class holds the state that LSR keeps for each use in IVUsers, as well
1276/// as uses invented by LSR itself. It includes information about what kinds of
1277/// things can be folded into the user, information about the user itself, and
1278/// information about how the use may be satisfied. TODO: Represent multiple
1279/// users of the same expression in common?
1280class LSRUse {
1281 DenseSet<SmallVector<const SCEV *, 4>> Uniquifier;
1282
1283public:
1284 /// An enum for a kind of use, indicating what types of scaled and immediate
1285 /// operands it might support.
1286 enum KindType {
1287 Basic, ///< A normal use, with no folding.
1288 Special, ///< A special case of basic, allowing -1 scales.
1289 Address, ///< An address use; folding according to TargetLowering
1290 ICmpZero ///< An equality icmp with both operands folded into one.
1291 // TODO: Add a generic icmp too?
1292 };
1293
1294 using SCEVUseKindPair = PointerIntPair<const SCEV *, 2, KindType>;
1295
1296 KindType Kind;
1297 MemAccessTy AccessTy;
1298
1299 /// The list of operands which are to be replaced.
1301
1302 /// Keep track of the min and max offsets of the fixups.
1303 Immediate MinOffset = Immediate::getFixedMax();
1304 Immediate MaxOffset = Immediate::getFixedMin();
1305
1306 /// This records whether all of the fixups using this LSRUse are outside of
1307 /// the loop, in which case some special-case heuristics may be used.
1308 bool AllFixupsOutsideLoop = true;
1309
1310 /// This records whether all of the fixups using this LSRUse are unconditional
1311 /// within the loop, meaning they will be executed on every path to the loop
1312 /// latch. This includes fixups before early exits.
1313 bool AllFixupsUnconditional = true;
1314
1315 /// RigidFormula is set to true to guarantee that this use will be associated
1316 /// with a single formula--the one that initially matched. Some SCEV
1317 /// expressions cannot be expanded. This allows LSR to consider the registers
1318 /// used by those expressions without the need to expand them later after
1319 /// changing the formula.
1320 bool RigidFormula = false;
1321
1322 /// A list of ways to build a value that can satisfy this user. After the
1323 /// list is populated, one of these is selected heuristically and used to
1324 /// formulate a replacement for OperandValToReplace in UserInst.
1325 SmallVector<Formula, 12> Formulae;
1326
1327 /// The set of register candidates used by all formulae in this LSRUse.
1328 SmallPtrSet<const SCEV *, 4> Regs;
1329
1330 LSRUse(KindType K, MemAccessTy AT) : Kind(K), AccessTy(AT) {}
1331
1332 LSRFixup &getNewFixup() {
1333 Fixups.push_back(LSRFixup());
1334 return Fixups.back();
1335 }
1336
1337 void pushFixup(LSRFixup &f) {
1338 Fixups.push_back(f);
1339 if (Immediate::isKnownGT(f.Offset, MaxOffset))
1340 MaxOffset = f.Offset;
1341 if (Immediate::isKnownLT(f.Offset, MinOffset))
1342 MinOffset = f.Offset;
1343 }
1344
1345 bool HasFormulaWithSameRegs(const Formula &F) const;
1346 float getNotSelectedProbability(const SCEV *Reg) const;
1347 bool InsertFormula(const Formula &F, const Loop &L);
1348 void DeleteFormula(Formula &F);
1349 void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses);
1350
1351 void print(raw_ostream &OS) const;
1352 void dump() const;
1353};
1354
1355} // end anonymous namespace
1356
1357static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1358 LSRUse::KindType Kind, MemAccessTy AccessTy,
1359 GlobalValue *BaseGV, Immediate BaseOffset,
1360 bool HasBaseReg, int64_t Scale,
1361 Instruction *Fixup = nullptr);
1362
1363static unsigned getSetupCost(const SCEV *Reg, unsigned Depth,
1364 const TargetTransformInfo &TTI) {
1365 if (isa<SCEVUnknown>(Reg))
1366 return 1;
1367 if (const auto *C = dyn_cast<SCEVConstant>(Reg)) {
1368 if (TTI.getIntImmCost(C->getAPInt(), C->getType(),
1371 return 0;
1372 return 1;
1373 }
1374 if (Depth == 0)
1375 return 0;
1376 if (const auto *S = dyn_cast<SCEVAddRecExpr>(Reg))
1377 return getSetupCost(S->getStart(), Depth - 1, TTI);
1378 if (auto S = dyn_cast<SCEVIntegralCastExpr>(Reg))
1379 return getSetupCost(S->getOperand(), Depth - 1, TTI);
1380 if (auto S = dyn_cast<SCEVNAryExpr>(Reg))
1381 return std::accumulate(S->operands().begin(), S->operands().end(), 0,
1382 [&](unsigned i, const SCEV *Reg) {
1383 return i + getSetupCost(Reg, Depth - 1, TTI);
1384 });
1385 if (auto S = dyn_cast<SCEVUDivExpr>(Reg))
1386 return getSetupCost(S->getLHS(), Depth - 1, TTI) +
1387 getSetupCost(S->getRHS(), Depth - 1, TTI);
1388 return 0;
1389}
1390
1391/// Tally up interesting quantities from the given register.
1392void Cost::RateRegister(const Formula &F, const SCEV *Reg,
1393 SmallPtrSetImpl<const SCEV *> &Regs, const LSRUse &LU,
1394 bool HardwareLoopProfitable) {
1395 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Reg)) {
1396 // If this is an addrec for another loop, it should be an invariant
1397 // with respect to L since L is the innermost loop (at least
1398 // for now LSR only handles innermost loops).
1399 if (AR->getLoop() != L) {
1400 // If the AddRec exists, consider it's register free and leave it alone.
1401 if (isExistingPhi(AR, *SE) && !(AMK & TTI::AMK_PostIndexed))
1402 return;
1403
1404 // It is bad to allow LSR for current loop to add induction variables
1405 // for its sibling loops.
1406 if (!AR->getLoop()->contains(L)) {
1407 Lose();
1408 return;
1409 }
1410
1411 // Otherwise, it will be an invariant with respect to Loop L.
1412 ++C.NumRegs;
1413 return;
1414 }
1415
1416 unsigned LoopCost = 1;
1417 if (LU.Kind == LSRUse::Address &&
1418 (TTI->isIndexedLoadLegal(TTI->MIM_PostInc, AR->getType()) ||
1419 TTI->isIndexedStoreLegal(TTI->MIM_PostInc, AR->getType()))) {
1420 const SCEV *Start;
1421 const APInt *Step;
1422 if (match(AR, m_scev_AffineAddRec(m_SCEV(Start), m_scev_APInt(Step)))) {
1423 // If the step size matches the base offset, we could use pre-indexed
1424 // addressing.
1425 bool CanPreIndex = (AMK & TTI::AMK_PreIndexed) &&
1426 F.BaseOffset.isFixed() &&
1427 *Step == F.BaseOffset.getFixedValue();
1428 bool CanPostIndex = (AMK & TTI::AMK_PostIndexed) &&
1429 !isa<SCEVConstant>(Start) &&
1430 SE->isLoopInvariant(Start, L);
1431 // We can only pre or post index when the load/store is unconditional.
1432 if ((CanPreIndex || CanPostIndex) && LU.AllFixupsUnconditional) {
1433 ++C.NumPrePostIndex;
1434 LoopCost = 0;
1435 }
1436 }
1437 }
1438
1439 // If the loop counts down to zero and we'll be using a hardware loop then
1440 // the addrec will be combined into the hardware loop instruction.
1441 if (LU.Kind == LSRUse::ICmpZero && F.countsDownToZero() &&
1442 HardwareLoopProfitable)
1443 LoopCost = 0;
1444 C.AddRecCost += LoopCost;
1445
1446 // Add the step value register, if it needs one.
1447 // TODO: The non-affine case isn't precisely modeled here.
1448 const SCEV *StepReg = AR->getOperand(1);
1449 if (!AR->isAffine() || !isa<SCEVConstant>(StepReg)) {
1450 // If the step amount is a constant multiplied by vscale then it can form
1451 // the immediate value of an add and doesn't use a register, so long as
1452 // the immediate value is legal.
1453 auto IsVScaleStep = [](const SCEV *Reg, const TargetTransformInfo *TTI) {
1454 const APInt *X;
1456 return false;
1457 return TTI->isLegalAddScalableImmediate(X->getLimitedValue());
1458 };
1459 if (!Regs.count(StepReg) && !IsVScaleStep(StepReg, TTI)) {
1460 RateRegister(F, StepReg, Regs, LU, HardwareLoopProfitable);
1461 if (isLoser())
1462 return;
1463 }
1464 }
1465 }
1466 ++C.NumRegs;
1467
1468 // Rough heuristic; favor registers which don't require extra setup
1469 // instructions in the preheader.
1470 C.SetupCost += getSetupCost(Reg, Opts->lsr_setupcost_depth_limit, *TTI);
1471 // Ensure we don't, even with the recusion limit, produce invalid costs.
1472 C.SetupCost = std::min<unsigned>(C.SetupCost, 1 << 16);
1473
1474 C.NumIVMuls += isa<SCEVMulExpr>(Reg) &&
1476}
1477
1478/// Record this register in the set. If we haven't seen it before, rate
1479/// it. Optional LoserRegs provides a way to declare any formula that refers to
1480/// one of those regs an instant loser.
1481void Cost::RatePrimaryRegister(const Formula &F, const SCEV *Reg,
1482 SmallPtrSetImpl<const SCEV *> &Regs,
1483 const LSRUse &LU, bool HardwareLoopProfitable,
1484 SmallPtrSetImpl<const SCEV *> *LoserRegs) {
1485 if (LoserRegs && LoserRegs->count(Reg)) {
1486 Lose();
1487 return;
1488 }
1489 if (Regs.insert(Reg).second) {
1490 RateRegister(F, Reg, Regs, LU, HardwareLoopProfitable);
1491 if (LoserRegs && isLoser())
1492 LoserRegs->insert(Reg);
1493 }
1494}
1495
1496void Cost::RateFormula(const Formula &F, SmallPtrSetImpl<const SCEV *> &Regs,
1497 const DenseSet<const SCEV *> &VisitedRegs,
1498 const LSRUse &LU, bool HardwareLoopProfitable,
1499 SmallPtrSetImpl<const SCEV *> *LoserRegs) {
1500 if (isLoser())
1501 return;
1502 assert(F.isCanonical(*L) && "Cost is accurate only for canonical formula");
1503 // Tally up the registers.
1504 unsigned PrevAddRecCost = C.AddRecCost;
1505 unsigned PrevNumRegs = C.NumRegs;
1506 unsigned PrevNumBaseAdds = C.NumBaseAdds;
1507 if (const SCEV *ScaledReg = F.ScaledReg) {
1508 if (VisitedRegs.count(ScaledReg)) {
1509 Lose();
1510 return;
1511 }
1512 RatePrimaryRegister(F, ScaledReg, Regs, LU, HardwareLoopProfitable,
1513 LoserRegs);
1514 if (isLoser())
1515 return;
1516 }
1517 for (const SCEV *BaseReg : F.BaseRegs) {
1518 if (VisitedRegs.count(BaseReg)) {
1519 Lose();
1520 return;
1521 }
1522 RatePrimaryRegister(F, BaseReg, Regs, LU, HardwareLoopProfitable,
1523 LoserRegs);
1524 if (isLoser())
1525 return;
1526 }
1527
1528 // Determine how many (unfolded) adds we'll need inside the loop.
1529 size_t NumBaseParts = F.getNumRegs();
1530 if (NumBaseParts > 1)
1531 // Do not count the base and a possible second register if the target
1532 // allows to fold 2 registers.
1533 C.NumBaseAdds +=
1534 NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(*TTI, LU, F)));
1535 C.NumBaseAdds += (F.UnfoldedOffset.isNonZero());
1536
1537 // Accumulate non-free scaling amounts.
1538 C.ScaleCost += getScalingFactorCost(*TTI, LU, F, *L).getValue();
1539
1540 // Tally up the non-zero immediates.
1541 for (const LSRFixup &Fixup : LU.Fixups) {
1542 if (Fixup.Offset.isCompatibleImmediate(F.BaseOffset)) {
1543 Immediate Offset = Fixup.Offset.addUnsigned(F.BaseOffset);
1544 if (F.BaseGV)
1545 C.ImmCost += 64; // Handle symbolic values conservatively.
1546 // TODO: This should probably be the pointer size.
1547 else if (Offset.isNonZero())
1548 C.ImmCost +=
1549 APInt(64, Offset.getKnownMinValue(), true).getSignificantBits();
1550
1551 // Check with target if this offset with this instruction is
1552 // specifically not supported.
1553 if (LU.Kind == LSRUse::Address && Offset.isNonZero() &&
1554 !isAMCompletelyFolded(*TTI, LSRUse::Address, LU.AccessTy, F.BaseGV,
1555 Offset, F.HasBaseReg, F.Scale, Fixup.UserInst))
1556 C.NumBaseAdds++;
1557 } else {
1558 // Incompatible immediate type, increase cost to avoid using
1559 C.ImmCost += 2048;
1560 }
1561 }
1562
1563 // If we don't count instruction cost exit here.
1564 if (!valueOr(Opts->lsr_insns_cost, true)) {
1565 assert(isValid() && "invalid cost");
1566 return;
1567 }
1568
1569 // Treat every new register that exceeds TTI.getNumberOfRegisters() - 1 as
1570 // additional instruction (at least fill).
1571 // TODO: Need distinguish register class?
1572 unsigned TTIRegNum = TTI->getNumberOfRegisters(
1573 TTI->getRegisterClassForType(false, F.getType())) - 1;
1574 if (C.NumRegs > TTIRegNum) {
1575 // Cost already exceeded TTIRegNum, then only newly added register can add
1576 // new instructions.
1577 if (PrevNumRegs > TTIRegNum)
1578 C.Insns += (C.NumRegs - PrevNumRegs);
1579 else
1580 C.Insns += (C.NumRegs - TTIRegNum);
1581 }
1582
1583 // If ICmpZero formula ends with not 0, it could not be replaced by
1584 // just add or sub. We'll need to compare final result of AddRec.
1585 // That means we'll need an additional instruction. But if the target can
1586 // macro-fuse a compare with a branch, don't count this extra instruction.
1587 // For -10 + {0, +, 1}:
1588 // i = i + 1;
1589 // cmp i, 10
1590 //
1591 // For {-10, +, 1}:
1592 // i = i + 1;
1593 if (LU.Kind == LSRUse::ICmpZero && !F.hasZeroEnd() &&
1594 !TTI->canMacroFuseCmp())
1595 C.Insns++;
1596 // Each new AddRec adds 1 instruction to calculation.
1597 C.Insns += (C.AddRecCost - PrevAddRecCost);
1598
1599 // BaseAdds adds instructions for unfolded registers.
1600 if (LU.Kind != LSRUse::ICmpZero)
1601 C.Insns += C.NumBaseAdds - PrevNumBaseAdds;
1602 assert(isValid() && "invalid cost");
1603}
1604
1605/// Set this cost to a losing value.
1606void Cost::Lose() {
1607 C.Insns = std::numeric_limits<unsigned>::max();
1608 C.NumRegs = std::numeric_limits<unsigned>::max();
1609 C.AddRecCost = std::numeric_limits<unsigned>::max();
1610 C.NumIVMuls = std::numeric_limits<unsigned>::max();
1611 C.NumBaseAdds = std::numeric_limits<unsigned>::max();
1612 C.ImmCost = std::numeric_limits<unsigned>::max();
1613 C.SetupCost = std::numeric_limits<unsigned>::max();
1614 C.ScaleCost = std::numeric_limits<unsigned>::max();
1615 C.NumPrePostIndex = std::numeric_limits<unsigned>::max();
1616}
1617
1618/// Choose the lower cost.
1619bool Cost::isLess(const Cost &Other) const {
1620 if (Opts->lsr_insns_cost == BoolOrDefault::True && C.Insns != Other.C.Insns)
1621 return C.Insns < Other.C.Insns;
1622 return TTI->isLSRCostLess(C, Other.C);
1623}
1624
1625#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1626void Cost::print(raw_ostream &OS) const {
1627 if (valueOr(Opts->lsr_insns_cost, true))
1628 OS << C.Insns << " instruction" << (C.Insns == 1 ? " " : "s ");
1629 OS << C.NumRegs << " reg" << (C.NumRegs == 1 ? "" : "s");
1630 if (C.AddRecCost != 0)
1631 OS << ", with addrec cost " << C.AddRecCost;
1632 if (C.NumIVMuls != 0)
1633 OS << ", plus " << C.NumIVMuls << " IV mul"
1634 << (C.NumIVMuls == 1 ? "" : "s");
1635 if (C.NumBaseAdds != 0)
1636 OS << ", plus " << C.NumBaseAdds << " base add"
1637 << (C.NumBaseAdds == 1 ? "" : "s");
1638 if (C.ScaleCost != 0)
1639 OS << ", plus " << C.ScaleCost << " scale cost";
1640 if (C.ImmCost != 0)
1641 OS << ", plus " << C.ImmCost << " imm cost";
1642 if (C.SetupCost != 0)
1643 OS << ", plus " << C.SetupCost << " setup cost";
1644 if (C.NumPrePostIndex != 0)
1645 OS << ", plus " << C.NumPrePostIndex << " pre/post-index";
1646}
1647
1648LLVM_DUMP_METHOD void Cost::dump() const {
1649 print(errs()); errs() << '\n';
1650}
1651#endif
1652
1653/// Test whether this fixup always uses its value outside of the given loop.
1654bool LSRFixup::isUseFullyOutsideLoop(const Loop *L) const {
1655 // PHI nodes use their value in their incoming blocks.
1656 if (const PHINode *PN = dyn_cast<PHINode>(UserInst)) {
1657 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
1658 if (PN->getIncomingValue(i) == OperandValToReplace &&
1659 L->contains(PN->getIncomingBlock(i)))
1660 return false;
1661 return true;
1662 }
1663
1664 return !L->contains(UserInst);
1665}
1666
1667#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1668void LSRFixup::print(raw_ostream &OS) const {
1669 OS << "UserInst=";
1670 // Store is common and interesting enough to be worth special-casing.
1671 if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) {
1672 OS << "store ";
1673 Store->getOperand(0)->printAsOperand(OS, /*PrintType=*/false);
1674 } else if (UserInst->getType()->isVoidTy())
1675 OS << UserInst->getOpcodeName();
1676 else
1677 UserInst->printAsOperand(OS, /*PrintType=*/false);
1678
1679 OS << ", OperandValToReplace=";
1680 OperandValToReplace->printAsOperand(OS, /*PrintType=*/false);
1681
1682 for (const Loop *PIL : PostIncLoops) {
1683 OS << ", PostIncLoop=";
1684 PIL->getHeader()->printAsOperand(OS, /*PrintType=*/false);
1685 }
1686
1687 if (Offset.isNonZero())
1688 OS << ", Offset=" << Offset;
1689}
1690
1691LLVM_DUMP_METHOD void LSRFixup::dump() const {
1692 print(errs()); errs() << '\n';
1693}
1694#endif
1695
1696/// Test whether this use as a formula which has the same registers as the given
1697/// formula.
1698bool LSRUse::HasFormulaWithSameRegs(const Formula &F) const {
1700 if (F.ScaledReg) Key.push_back(F.ScaledReg);
1701 // Unstable sort by host order ok, because this is only used for uniquifying.
1702 llvm::sort(Key);
1703 return Uniquifier.count(Key);
1704}
1705
1706/// The function returns a probability of selecting formula without Reg.
1707float LSRUse::getNotSelectedProbability(const SCEV *Reg) const {
1708 unsigned FNum = 0;
1709 for (const Formula &F : Formulae)
1710 if (F.referencesReg(Reg))
1711 FNum++;
1712 return ((float)(Formulae.size() - FNum)) / Formulae.size();
1713}
1714
1715/// If the given formula has not yet been inserted, add it to the list, and
1716/// return true. Return false otherwise. The formula must be in canonical form.
1717bool LSRUse::InsertFormula(const Formula &F, const Loop &L) {
1718 assert(F.isCanonical(L) && "Invalid canonical representation");
1719
1720 if (!Formulae.empty() && RigidFormula)
1721 return false;
1722
1724 if (F.ScaledReg) Key.push_back(F.ScaledReg);
1725 // Unstable sort by host order ok, because this is only used for uniquifying.
1726 llvm::sort(Key, std::less<>());
1727
1728 if (!Uniquifier.insert(Key).second)
1729 return false;
1730
1731 // Using a register to hold the value of 0 is not profitable.
1732 assert((!F.ScaledReg || !F.ScaledReg->isZero()) &&
1733 "Zero allocated in a scaled register!");
1734#ifndef NDEBUG
1735 for (const SCEV *BaseReg : F.BaseRegs)
1736 assert(!BaseReg->isZero() && "Zero allocated in a base register!");
1737#endif
1738
1739 // Add the formula to the list.
1740 Formulae.push_back(F);
1741
1742 // Record registers now being used by this use.
1743 Regs.insert_range(F.BaseRegs);
1744 if (F.ScaledReg)
1745 Regs.insert(F.ScaledReg);
1746
1747 return true;
1748}
1749
1750/// Remove the given formula from this use's list.
1751void LSRUse::DeleteFormula(Formula &F) {
1752 if (&F != &Formulae.back())
1753 std::swap(F, Formulae.back());
1754 Formulae.pop_back();
1755}
1756
1757/// Recompute the Regs field, and update RegUses.
1758void LSRUse::RecomputeRegs(size_t LUIdx, RegUseTracker &RegUses) {
1759 // Now that we've filtered out some formulae, recompute the Regs set.
1760 SmallPtrSet<const SCEV *, 4> OldRegs = std::move(Regs);
1761 Regs.clear();
1762 for (const Formula &F : Formulae) {
1763 if (F.ScaledReg) Regs.insert(F.ScaledReg);
1764 Regs.insert_range(F.BaseRegs);
1765 }
1766
1767 // Update the RegTracker.
1768 for (const SCEV *S : OldRegs)
1769 if (!Regs.count(S))
1770 RegUses.dropRegister(S, LUIdx);
1771}
1772
1773#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1774void LSRUse::print(raw_ostream &OS) const {
1775 OS << "LSR Use: Kind=";
1776 switch (Kind) {
1777 case Basic: OS << "Basic"; break;
1778 case Special: OS << "Special"; break;
1779 case ICmpZero: OS << "ICmpZero"; break;
1780 case Address:
1781 OS << "Address of ";
1782 if (AccessTy.MemTy->isPointerTy())
1783 OS << "pointer"; // the full pointer type could be really verbose
1784 else {
1785 OS << *AccessTy.MemTy;
1786 }
1787
1788 OS << " in addrspace(" << AccessTy.AddrSpace << ')';
1789 }
1790
1791 OS << ", Offsets={";
1792 bool NeedComma = false;
1793 for (const LSRFixup &Fixup : Fixups) {
1794 if (NeedComma) OS << ',';
1795 OS << Fixup.Offset;
1796 NeedComma = true;
1797 }
1798 OS << '}';
1799
1800 if (AllFixupsOutsideLoop)
1801 OS << ", all-fixups-outside-loop";
1802
1803 if (AllFixupsUnconditional)
1804 OS << ", all-fixups-unconditional";
1805}
1806
1807LLVM_DUMP_METHOD void LSRUse::dump() const {
1808 print(errs()); errs() << '\n';
1809}
1810#endif
1811
1813 LSRUse::KindType Kind, MemAccessTy AccessTy,
1814 GlobalValue *BaseGV, Immediate BaseOffset,
1815 bool HasBaseReg, int64_t Scale,
1816 Instruction *Fixup /* = nullptr */) {
1817 switch (Kind) {
1818 case LSRUse::Address: {
1819 int64_t FixedOffset =
1820 BaseOffset.isScalable() ? 0 : BaseOffset.getFixedValue();
1821 int64_t ScalableOffset =
1822 BaseOffset.isScalable() ? BaseOffset.getKnownMinValue() : 0;
1823 return TTI.isLegalAddressingMode(AccessTy.MemTy, BaseGV, FixedOffset,
1824 HasBaseReg, Scale, AccessTy.AddrSpace,
1825 Fixup, ScalableOffset);
1826 }
1827 case LSRUse::ICmpZero:
1828 // There's not even a target hook for querying whether it would be legal to
1829 // fold a GV into an ICmp.
1830 if (BaseGV)
1831 return false;
1832
1833 // ICmp only has two operands; don't allow more than two non-trivial parts.
1834 if (Scale != 0 && HasBaseReg && BaseOffset.isNonZero())
1835 return false;
1836
1837 // ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale by
1838 // putting the scaled register in the other operand of the icmp.
1839 if (Scale != 0 && Scale != -1)
1840 return false;
1841
1842 // If we have low-level target information, ask the target if it can fold an
1843 // integer immediate on an icmp.
1844 if (BaseOffset.isNonZero()) {
1845 // We don't have an interface to query whether the target supports
1846 // icmpzero against scalable quantities yet.
1847 if (BaseOffset.isScalable())
1848 return false;
1849
1850 // We have one of:
1851 // ICmpZero BaseReg + BaseOffset => ICmp BaseReg, -BaseOffset
1852 // ICmpZero -1*ScaleReg + BaseOffset => ICmp ScaleReg, BaseOffset
1853 // Offs is the ICmp immediate.
1854 if (Scale == 0)
1855 // The cast does the right thing with
1856 // std::numeric_limits<int64_t>::min().
1857 BaseOffset = BaseOffset.getFixed(-(uint64_t)BaseOffset.getFixedValue());
1858 return TTI.isLegalICmpImmediate(BaseOffset.getFixedValue());
1859 }
1860
1861 // ICmpZero BaseReg + -1*ScaleReg => ICmp BaseReg, ScaleReg
1862 return true;
1863
1864 case LSRUse::Basic:
1865 // Only handle single-register values.
1866 return !BaseGV && Scale == 0 && BaseOffset.isZero();
1867
1868 case LSRUse::Special:
1869 // Special case Basic to handle -1 scales.
1870 return !BaseGV && (Scale == 0 || Scale == -1) && BaseOffset.isZero();
1871 }
1872
1873 llvm_unreachable("Invalid LSRUse Kind!");
1874}
1875
1877 Immediate MinOffset, Immediate MaxOffset,
1878 LSRUse::KindType Kind, MemAccessTy AccessTy,
1879 GlobalValue *BaseGV, Immediate BaseOffset,
1880 bool HasBaseReg, int64_t Scale) {
1881 if (BaseOffset.isNonZero() &&
1882 (BaseOffset.isScalable() != MinOffset.isScalable() ||
1883 BaseOffset.isScalable() != MaxOffset.isScalable()))
1884 return false;
1885 // Check for overflow.
1886 int64_t Base = BaseOffset.getKnownMinValue();
1887 int64_t Min = MinOffset.getKnownMinValue();
1888 int64_t Max = MaxOffset.getKnownMinValue();
1889 if (((int64_t)((uint64_t)Base + Min) > Base) != (Min > 0))
1890 return false;
1891 MinOffset = Immediate::get((uint64_t)Base + Min, MinOffset.isScalable());
1892 if (((int64_t)((uint64_t)Base + Max) > Base) != (Max > 0))
1893 return false;
1894 MaxOffset = Immediate::get((uint64_t)Base + Max, MaxOffset.isScalable());
1895
1896 return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, MinOffset,
1897 HasBaseReg, Scale) &&
1898 isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, MaxOffset,
1899 HasBaseReg, Scale);
1900}
1901
1903 Immediate MinOffset, Immediate MaxOffset,
1904 LSRUse::KindType Kind, MemAccessTy AccessTy,
1905 const Formula &F, const Loop &L) {
1906 // For the purpose of isAMCompletelyFolded either having a canonical formula
1907 // or a scale not equal to zero is correct.
1908 // Problems may arise from non canonical formulae having a scale == 0.
1909 // Strictly speaking it would best to just rely on canonical formulae.
1910 // However, when we generate the scaled formulae, we first check that the
1911 // scaling factor is profitable before computing the actual ScaledReg for
1912 // compile time sake.
1913 assert((F.isCanonical(L) || F.Scale != 0));
1914 return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
1915 F.BaseGV, F.BaseOffset, F.HasBaseReg, F.Scale);
1916}
1917
1918/// Test whether we know how to expand the current formula.
1919static bool isLegalUse(const TargetTransformInfo &TTI, Immediate MinOffset,
1920 Immediate MaxOffset, LSRUse::KindType Kind,
1921 MemAccessTy AccessTy, GlobalValue *BaseGV,
1922 Immediate BaseOffset, bool HasBaseReg, int64_t Scale) {
1923 // We know how to expand completely foldable formulae.
1924 return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
1925 BaseOffset, HasBaseReg, Scale) ||
1926 // Or formulae that use a base register produced by a sum of base
1927 // registers.
1928 (Scale == 1 &&
1929 isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
1930 BaseGV, BaseOffset, true, 0));
1931}
1932
1933static bool isLegalUse(const TargetTransformInfo &TTI, Immediate MinOffset,
1934 Immediate MaxOffset, LSRUse::KindType Kind,
1935 MemAccessTy AccessTy, const Formula &F) {
1936 return isLegalUse(TTI, MinOffset, MaxOffset, Kind, AccessTy, F.BaseGV,
1937 F.BaseOffset, F.HasBaseReg, F.Scale);
1938}
1939
1941 Immediate Offset) {
1942 if (Offset.isScalable())
1943 return TTI.isLegalAddScalableImmediate(Offset.getKnownMinValue());
1944
1945 return TTI.isLegalAddImmediate(Offset.getFixedValue());
1946}
1947
1949 const LSRUse &LU, const Formula &F) {
1950 // Target may want to look at the user instructions.
1951 if (LU.Kind == LSRUse::Address && TTI.LSRWithInstrQueries()) {
1952 for (const LSRFixup &Fixup : LU.Fixups)
1953 if (!isAMCompletelyFolded(TTI, LSRUse::Address, LU.AccessTy, F.BaseGV,
1954 (F.BaseOffset + Fixup.Offset), F.HasBaseReg,
1955 F.Scale, Fixup.UserInst))
1956 return false;
1957 return true;
1958 }
1959
1960 return isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1961 LU.AccessTy, F.BaseGV, F.BaseOffset, F.HasBaseReg,
1962 F.Scale);
1963}
1964
1966 const LSRUse &LU, const Formula &F,
1967 const Loop &L) {
1968 if (!F.Scale)
1969 return 0;
1970
1971 // If the use is not completely folded in that instruction, we will have to
1972 // pay an extra cost only for scale != 1.
1973 if (!isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
1974 LU.AccessTy, F, L))
1975 return F.Scale != 1;
1976
1977 switch (LU.Kind) {
1978 case LSRUse::Address: {
1979 // Check the scaling factor cost with both the min and max offsets.
1980 int64_t ScalableMin = 0, ScalableMax = 0, FixedMin = 0, FixedMax = 0;
1981 if (F.BaseOffset.isScalable()) {
1982 ScalableMin = (F.BaseOffset + LU.MinOffset).getKnownMinValue();
1983 ScalableMax = (F.BaseOffset + LU.MaxOffset).getKnownMinValue();
1984 } else {
1985 FixedMin = (F.BaseOffset + LU.MinOffset).getFixedValue();
1986 FixedMax = (F.BaseOffset + LU.MaxOffset).getFixedValue();
1987 }
1988 InstructionCost ScaleCostMinOffset = TTI.getScalingFactorCost(
1989 LU.AccessTy.MemTy, F.BaseGV, StackOffset::get(FixedMin, ScalableMin),
1990 F.HasBaseReg, F.Scale, LU.AccessTy.AddrSpace);
1991 InstructionCost ScaleCostMaxOffset = TTI.getScalingFactorCost(
1992 LU.AccessTy.MemTy, F.BaseGV, StackOffset::get(FixedMax, ScalableMax),
1993 F.HasBaseReg, F.Scale, LU.AccessTy.AddrSpace);
1994
1995 assert(ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() &&
1996 "Legal addressing mode has an illegal cost!");
1997 return std::max(ScaleCostMinOffset, ScaleCostMaxOffset);
1998 }
1999 case LSRUse::ICmpZero:
2000 case LSRUse::Basic:
2001 case LSRUse::Special:
2002 // The use is completely folded, i.e., everything is folded into the
2003 // instruction.
2004 return 0;
2005 }
2006
2007 llvm_unreachable("Invalid LSRUse Kind!");
2008}
2009
2010static bool isAlwaysFoldable(const ScalarOptions &Opts,
2011 const TargetTransformInfo &TTI,
2012 LSRUse::KindType Kind, MemAccessTy AccessTy,
2013 GlobalValue *BaseGV, Immediate BaseOffset,
2014 bool HasBaseReg) {
2015 // Fast-path: zero is always foldable.
2016 if (BaseOffset.isZero() && !BaseGV)
2017 return true;
2018
2019 // Conservatively, create an address with an immediate and a
2020 // base and a scale.
2021 int64_t Scale = Kind == LSRUse::ICmpZero ? -1 : 1;
2022
2023 // Canonicalize a scale of 1 to a base register if the formula doesn't
2024 // already have a base register.
2025 if (!HasBaseReg && Scale == 1) {
2026 Scale = 0;
2027 HasBaseReg = true;
2028 }
2029
2030 // FIXME: Try with + without a scale? Maybe based on TTI?
2031 // I think basereg + scaledreg + immediateoffset isn't a good 'conservative'
2032 // default for many architectures, not just AArch64 SVE. More investigation
2033 // needed later to determine if this should be used more widely than just
2034 // on scalable types.
2035 if (HasBaseReg && BaseOffset.isNonZero() && Kind != LSRUse::ICmpZero &&
2036 AccessTy.MemTy && AccessTy.MemTy->isScalableTy() &&
2037 Opts.lsr_drop_scaled_reg_for_vscale)
2038 Scale = 0;
2039
2040 return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset,
2041 HasBaseReg, Scale);
2042}
2043
2044static bool isAlwaysFoldable(const ScalarOptions &Opts,
2045 const TargetTransformInfo &TTI,
2046 ScalarEvolution &SE, Immediate MinOffset,
2047 Immediate MaxOffset, LSRUse::KindType Kind,
2048 MemAccessTy AccessTy, const SCEV *S,
2049 bool HasBaseReg) {
2050 // Fast-path: zero is always foldable.
2051 if (S->isZero()) return true;
2052
2053 // Conservatively, create an address with an immediate and a
2054 // base and a scale.
2055 SCEVUse SCopy = S;
2056 Immediate BaseOffset = extractImmediate(Opts, SCopy, SE);
2057 GlobalValue *BaseGV = ExtractSymbol(SCopy, SE);
2058
2059 // If there's anything else involved, it's not foldable.
2060 if (!SCopy->isZero())
2061 return false;
2062
2063 // Fast-path: zero is always foldable.
2064 if (BaseOffset.isZero() && !BaseGV)
2065 return true;
2066
2067 if (BaseOffset.isScalable())
2068 return false;
2069
2070 // Conservatively, create an address with an immediate and a
2071 // base and a scale.
2072 int64_t Scale = Kind == LSRUse::ICmpZero ? -1 : 1;
2073
2074 return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
2075 BaseOffset, HasBaseReg, Scale);
2076}
2077
2078namespace {
2079
2080/// An individual increment in a Chain of IV increments. Relate an IV user to
2081/// an expression that computes the IV it uses from the IV used by the previous
2082/// link in the Chain.
2083///
2084/// For the head of a chain, IncExpr holds the absolute SCEV expression for the
2085/// original IVOperand. The head of the chain's IVOperand is only valid during
2086/// chain collection, before LSR replaces IV users. During chain generation,
2087/// IncExpr can be used to find the new IVOperand that computes the same
2088/// expression.
2089struct IVInc {
2090 Instruction *UserInst;
2091 Value* IVOperand;
2092 const SCEV *IncExpr;
2093
2094 IVInc(Instruction *U, Value *O, const SCEV *E)
2095 : UserInst(U), IVOperand(O), IncExpr(E) {}
2096};
2097
2098// The list of IV increments in program order. We typically add the head of a
2099// chain without finding subsequent links.
2100struct IVChain {
2102 const SCEV *ExprBase = nullptr;
2103
2104 IVChain() = default;
2105 IVChain(const IVInc &Head, const SCEV *Base)
2106 : Incs(1, Head), ExprBase(Base) {}
2107
2108 using const_iterator = SmallVectorImpl<IVInc>::const_iterator;
2109
2110 // Return the first increment in the chain.
2111 const_iterator begin() const {
2112 assert(!Incs.empty());
2113 return std::next(Incs.begin());
2114 }
2115 const_iterator end() const {
2116 return Incs.end();
2117 }
2118
2119 // Returns true if this chain contains any increments.
2120 bool hasIncs() const { return Incs.size() >= 2; }
2121
2122 // Add an IVInc to the end of this chain.
2123 void add(const IVInc &X) { Incs.push_back(X); }
2124
2125 // Returns the last UserInst in the chain.
2126 Instruction *tailUserInst() const { return Incs.back().UserInst; }
2127
2128 // Returns true if IncExpr can be profitably added to this chain.
2129 bool isProfitableIncrement(const SCEV *OperExpr,
2130 const SCEV *IncExpr,
2131 ScalarEvolution&);
2132};
2133
2134/// Helper for CollectChains to track multiple IV increment uses. Distinguish
2135/// between FarUsers that definitely cross IV increments and NearUsers that may
2136/// be used between IV increments.
2137struct ChainUsers {
2138 SmallPtrSet<Instruction*, 4> FarUsers;
2139 SmallPtrSet<Instruction*, 4> NearUsers;
2140};
2141
2142/// This class holds state for the main loop strength reduction logic.
2143class LSRInstance {
2144 const ScalarOptions &Opts;
2145 IVUsers &IU;
2146 ScalarEvolution &SE;
2147 DominatorTree &DT;
2148 LoopInfo &LI;
2149 AssumptionCache &AC;
2150 TargetLibraryInfo &TLI;
2151 const TargetTransformInfo &TTI;
2152 Loop *const L;
2153 MemorySSAUpdater *MSSAU;
2155 mutable SCEVExpander Rewriter;
2156 bool Changed = false;
2157 bool HardwareLoopProfitable = false;
2158 bool ShouldPreserveLCSSA = false;
2159
2160 /// This is the insert position that the current loop's induction variable
2161 /// increment should be placed. In simple loops, this is the latch block's
2162 /// terminator. But in more complicated cases, this is a position which will
2163 /// dominate all the in-loop post-increment users.
2164 Instruction *IVIncInsertPos = nullptr;
2165
2166 /// Interesting factors between use strides.
2167 ///
2168 /// We explicitly use a SetVector which contains a SmallSet, instead of the
2169 /// default, a SmallDenseSet, because we need to use the full range of
2170 /// int64_ts, and there's currently no good way of doing that with
2171 /// SmallDenseSet.
2172 SetVector<int64_t, SmallVector<int64_t, 8>, SmallSet<int64_t, 8>> Factors;
2173
2174 /// The cost of the current SCEV, the best solution by LSR will be dropped if
2175 /// the solution is not profitable.
2176 Cost BaselineCost;
2177
2178 /// Interesting use types, to facilitate truncation reuse.
2179 SmallSetVector<Type *, 4> Types;
2180
2181 /// The list of interesting uses.
2183
2184 /// Track which uses use which register candidates.
2185 RegUseTracker RegUses;
2186
2187 // Limit the number of chains to avoid quadratic behavior. We don't expect to
2188 // have more than a few IV increment chains in a loop. Missing a Chain falls
2189 // back to normal LSR behavior for those uses.
2190 static const unsigned MaxChains = 8;
2191
2192 /// IV users can form a chain of IV increments.
2194
2195 /// IV users that belong to profitable IVChains.
2196 SmallPtrSet<Use*, MaxChains> IVIncSet;
2197
2198 /// Induction variables that were generated and inserted by the SCEV Expander.
2199 SmallVector<llvm::WeakVH, 2> ScalarEvolutionIVs;
2200
2201 // Inserting instructions in the loop and using them as PHI's input could
2202 // break LCSSA in case if PHI's parent block is not a loop exit (i.e. the
2203 // corresponding incoming block is not loop exiting). So collect all such
2204 // instructions to form LCSSA for them later.
2205 SmallSetVector<Instruction *, 4> InsertedNonLCSSAInsts;
2206
2207 void OptimizeShadowIV();
2208 bool FindIVUserForCond(Instruction *Cond, IVStrideUse *&CondUse);
2209 Instruction *OptimizeMax(ICmpInst *Cond, IVStrideUse *&CondUse);
2210 ICmpInst *OptimizeLoopTermCondToEquality(ICmpInst *Cond,
2211 IVStrideUse *&CondUse,
2212 BasicBlock *ExitingBlock);
2213 void OptimizeLoopTermCond();
2214
2215 void ChainInstruction(Instruction *UserInst, Instruction *IVOper,
2216 SmallVectorImpl<ChainUsers> &ChainUsersVec);
2217 void FinalizeChain(IVChain &Chain);
2218 void CollectChains();
2219 void GenerateIVChain(const IVChain &Chain,
2220 SmallVectorImpl<WeakTrackingVH> &DeadInsts);
2221
2222 void CollectInterestingTypesAndFactors();
2223 void CollectFixupsAndInitialFormulae();
2224
2225 // Support for sharing of LSRUses between LSRFixups.
2226 using UseMapTy = DenseMap<LSRUse::SCEVUseKindPair, size_t>;
2227 UseMapTy UseMap;
2228
2229 bool reconcileNewOffset(LSRUse &LU, Immediate NewOffset, bool HasBaseReg,
2230 LSRUse::KindType Kind, MemAccessTy AccessTy);
2231
2232 std::pair<size_t, Immediate> getUse(const SCEV *&Expr, LSRUse::KindType Kind,
2233 MemAccessTy AccessTy);
2234
2235 void DeleteUse(LSRUse &LU, size_t LUIdx);
2236
2237 LSRUse *FindUseWithSimilarFormula(const Formula &F, const LSRUse &OrigLU);
2238
2239 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2240 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2241 void CountRegisters(const Formula &F, size_t LUIdx);
2242 bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
2243 bool IsFixupExecutedEachIncrement(const LSRFixup &LF) const;
2244
2245 void CollectLoopInvariantFixupsAndFormulae();
2246
2247 void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
2248 unsigned Depth = 0);
2249
2250 void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
2251 const Formula &Base, unsigned Depth,
2252 size_t Idx, bool IsScaledReg = false);
2253 void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
2254 void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2255 const Formula &Base, size_t Idx,
2256 bool IsScaledReg = false);
2257 void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2258 void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2259 const Formula &Base,
2260 const SmallVectorImpl<Immediate> &Worklist,
2261 size_t Idx, bool IsScaledReg = false);
2262 void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2263 void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2264 void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2265 void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
2266 void GenerateCrossUseConstantOffsets();
2267 void GenerateAllReuseFormulae();
2268
2269 void FilterOutUndesirableDedicatedRegisters();
2270
2271 size_t EstimateSearchSpaceComplexity() const;
2272 void NarrowSearchSpaceByDetectingSupersets();
2273 void NarrowSearchSpaceByCollapsingUnrolledCode();
2274 void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
2275 void NarrowSearchSpaceByFilterFormulaWithSameScaledReg();
2276 void NarrowSearchSpaceByFilterPostInc();
2277 void NarrowSearchSpaceByMergingUsesOutsideLoop();
2278 void NarrowSearchSpaceByDeletingCostlyFormulas();
2279 void NarrowSearchSpaceByPickingWinnerRegs();
2280 void NarrowSearchSpaceUsingHeuristics();
2281
2282 void SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
2283 Cost &SolutionCost,
2284 SmallVectorImpl<const Formula *> &Workspace,
2285 const Cost &CurCost,
2286 const SmallPtrSet<const SCEV *, 16> &CurRegs,
2287 DenseSet<const SCEV *> &VisitedRegs) const;
2288 void Solve(SmallVectorImpl<const Formula *> &Solution) const;
2289
2291 HoistInsertPosition(BasicBlock::iterator IP,
2292 const SmallVectorImpl<Instruction *> &Inputs) const;
2293 BasicBlock::iterator AdjustInsertPositionForExpand(BasicBlock::iterator IP,
2294 const LSRFixup &LF,
2295 const LSRUse &LU) const;
2296
2297 Value *Expand(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2299 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const;
2300 void RewriteForPHI(PHINode *PN, const LSRUse &LU, const LSRFixup &LF,
2301 const Formula &F,
2302 SmallVectorImpl<WeakTrackingVH> &DeadInsts);
2303 void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2304 SmallVectorImpl<WeakTrackingVH> &DeadInsts);
2305 void ImplementSolution(const SmallVectorImpl<const Formula *> &Solution);
2306
2307public:
2308 // TODO(boomanaiden154): The PreserveLCSSA flag is a hack to allow
2309 // experimentation with the NewPM which requires LCSSA preservation while
2310 // some of the details are worked out in LSR. Eventually it should be set
2311 // to true and removed.
2312 LSRInstance(const ScalarOptions &Opts, Loop *L, IVUsers &IU,
2313 ScalarEvolution &SE, DominatorTree &DT, LoopInfo &LI,
2314 const TargetTransformInfo &TTI, AssumptionCache &AC,
2315 TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU,
2316 bool PreserveLCSSA);
2317
2318 bool getChanged() const { return Changed; }
2319 const SmallVectorImpl<WeakVH> &getScalarEvolutionIVs() const {
2320 return ScalarEvolutionIVs;
2321 }
2322
2323 void print_factors_and_types(raw_ostream &OS) const;
2324 void print_fixups(raw_ostream &OS) const;
2325 void print_uses(raw_ostream &OS) const;
2326 void print(raw_ostream &OS) const;
2327 void dump() const;
2328};
2329
2330} // end anonymous namespace
2331
2332/// If IV is used in a int-to-float cast inside the loop then try to eliminate
2333/// the cast operation.
2334void LSRInstance::OptimizeShadowIV() {
2335 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2336 if (isa<SCEVCouldNotCompute>(BackedgeTakenCount))
2337 return;
2338
2339 for (IVUsers::const_iterator UI = IU.begin(), E = IU.end();
2340 UI != E; /* empty */) {
2341 IVUsers::const_iterator CandidateUI = UI;
2342 ++UI;
2343 Instruction *ShadowUse = CandidateUI->getUser();
2344 Type *DestTy = nullptr;
2345 bool IsSigned = false;
2346
2347 /* If shadow use is a int->float cast then insert a second IV
2348 to eliminate this cast.
2349
2350 for (unsigned i = 0; i < n; ++i)
2351 foo((double)i);
2352
2353 is transformed into
2354
2355 double d = 0.0;
2356 for (unsigned i = 0; i < n; ++i, ++d)
2357 foo(d);
2358 */
2359 if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser())) {
2360 IsSigned = false;
2361 DestTy = UCast->getDestTy();
2362 }
2363 else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser())) {
2364 IsSigned = true;
2365 DestTy = SCast->getDestTy();
2366 }
2367 if (!DestTy) continue;
2368
2369 // If target does not support DestTy natively then do not apply
2370 // this transformation.
2371 if (!TTI.isTypeLegal(DestTy)) continue;
2372
2373 PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0));
2374 if (!PH) continue;
2375 if (PH->getNumIncomingValues() != 2) continue;
2376
2377 // If the calculation in integers overflows, the result in FP type will
2378 // differ. So we only can do this transformation if we are guaranteed to not
2379 // deal with overflowing values
2380 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(PH));
2381 if (!AR) continue;
2382 if (IsSigned && !AR->hasNoSignedWrap()) continue;
2383 if (!IsSigned && !AR->hasNoUnsignedWrap()) continue;
2384
2385 Type *SrcTy = PH->getType();
2386 int Mantissa = DestTy->getFPMantissaWidth();
2387 if (Mantissa == -1) continue;
2388 if ((int)SE.getTypeSizeInBits(SrcTy) > Mantissa)
2389 continue;
2390
2391 unsigned Entry, Latch;
2392 if (PH->getIncomingBlock(0) == L->getLoopPreheader()) {
2393 Entry = 0;
2394 Latch = 1;
2395 } else {
2396 Entry = 1;
2397 Latch = 0;
2398 }
2399
2400 ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry));
2401 if (!Init) continue;
2402 Constant *NewInit = ConstantFP::get(DestTy, IsSigned ?
2403 (double)Init->getSExtValue() :
2404 (double)Init->getZExtValue());
2405
2406 BinaryOperator *Incr =
2408 if (!Incr) continue;
2409 if (Incr->getOpcode() != Instruction::Add
2410 && Incr->getOpcode() != Instruction::Sub)
2411 continue;
2412
2413 /* Initialize new IV, double d = 0.0 in above example. */
2414 ConstantInt *C = nullptr;
2415 if (Incr->getOperand(0) == PH)
2417 else if (Incr->getOperand(1) == PH)
2419 else
2420 continue;
2421
2422 if (!C) continue;
2423
2424 // Ignore negative constants, as the code below doesn't handle them
2425 // correctly. TODO: Remove this restriction.
2426 if (!C->getValue().isStrictlyPositive())
2427 continue;
2428
2429 /* Add new PHINode. */
2430 PHINode *NewPH = PHINode::Create(DestTy, 2, "IV.S.", PH->getIterator());
2431 NewPH->setDebugLoc(PH->getDebugLoc());
2432
2433 /* create new increment. '++d' in above example. */
2434 Constant *CFP = ConstantFP::get(DestTy, C->getZExtValue());
2435 BinaryOperator *NewIncr = BinaryOperator::Create(
2436 Incr->getOpcode() == Instruction::Add ? Instruction::FAdd
2437 : Instruction::FSub,
2438 NewPH, CFP, "IV.S.next.", Incr->getIterator());
2439 NewIncr->setDebugLoc(Incr->getDebugLoc());
2440
2441 NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry));
2442 NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch));
2443
2444 /* Remove cast operation */
2445 ShadowUse->replaceAllUsesWith(NewPH);
2446 ShadowUse->eraseFromParent();
2447 Changed = true;
2448 break;
2449 }
2450}
2451
2452/// If Cond has an operand that is an expression of an IV, set the IV user and
2453/// stride information and return true, otherwise return false.
2454bool LSRInstance::FindIVUserForCond(Instruction *Cond, IVStrideUse *&CondUse) {
2455 for (IVStrideUse &U : IU)
2456 if (U.getUser() == Cond) {
2457 // NOTE: we could handle setcc instructions with multiple uses here, but
2458 // InstCombine does it as well for simple uses, it's not clear that it
2459 // occurs enough in real life to handle.
2460 CondUse = &U;
2461 return true;
2462 }
2463 return false;
2464}
2465
2466/// Replace a relational loop exit with an equality comparison when the target
2467/// prefers the exact exit value as an equality immediate.
2468ICmpInst *LSRInstance::OptimizeLoopTermCondToEquality(
2469 ICmpInst *Cond, IVStrideUse *&CondUse, BasicBlock *ExitingBlock) {
2470 if (Cond->isEquality() || ExitingBlock != L->getLoopLatch())
2471 return Cond;
2472
2473 auto *IV = dyn_cast<PHINode>(CondUse->getOperandValToReplace());
2474 if (!IV || !IV->getType()->isIntegerTy() || IV->getParent() != L->getHeader())
2475 return Cond;
2476
2477 const auto *AR = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(IV));
2478 if (!AR || AR->getLoop() != L || !AR->isAffine())
2479 return Cond;
2480
2481 const auto *Step = dyn_cast<SCEVConstant>(AR->getStepRecurrence(SE));
2482 if (!Step || Step->isZero())
2483 return Cond;
2484
2485 const SCEV *ExitCount = SE.getExitCount(L, ExitingBlock);
2486 if (isa<SCEVCouldNotCompute>(ExitCount))
2487 return Cond;
2488
2489 // A W-bit add recurrence with step S has period
2490 // 2^W / gcd(2^W, S) = 2^(W - countr_zero(S)). Require the exit count to be
2491 // smaller without relying on poison-derived no-wrap flags, so equality
2492 // cannot match on an earlier iteration.
2493 const APInt &StepValue = Step->getAPInt();
2494 unsigned RecurrencePeriodLog2 =
2495 StepValue.getBitWidth() - StepValue.countr_zero();
2496 if (!SE.getUnsignedRangeMax(ExitCount).isIntN(RecurrencePeriodLog2))
2497 return Cond;
2498
2499 const auto *ExitValue =
2500 dyn_cast<SCEVConstant>(AR->evaluateAtIteration(ExitCount, SE));
2501 if (!ExitValue || !TTI.preferLoopExitWithEqualityComparison(
2502 *Cond, ExitValue->getAPInt() + StepValue))
2503 return Cond;
2504
2505 LLVM_DEBUG(dbgs() << " Rewriting loop exit to compare " << *IV << " against "
2506 << *ExitValue << '\n');
2507
2508 auto *TermBr = cast<CondBrInst>(ExitingBlock->getTerminator());
2509 bool TrueContinues = L->contains(TermBr->getSuccessor(0));
2510 assert(TrueContinues != L->contains(TermBr->getSuccessor(1)) &&
2511 "Expected exactly one in-loop successor");
2512
2513 auto Pred = TrueContinues ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
2514 auto *NewCond = new ICmpInst(TermBr->getIterator(), Pred, IV,
2515 ExitValue->getValue(), "lsr.exitcond");
2516 NewCond->setDebugLoc(Cond->getDebugLoc());
2517 TermBr->setCondition(NewCond);
2518
2519 if (Cond->use_empty()) {
2520 CondUse->setUser(NewCond);
2521 Cond->eraseFromParent();
2522 } else {
2523 CondUse = &IU.AddUser(NewCond, IV);
2524 }
2525
2526 Changed = true;
2527 return NewCond;
2528}
2529
2530/// Rewrite the loop's terminating condition if it uses a max computation.
2531///
2532/// This is a narrow solution to a specific, but acute, problem. For loops
2533/// like this:
2534///
2535/// i = 0;
2536/// do {
2537/// p[i] = 0.0;
2538/// } while (++i < n);
2539///
2540/// the trip count isn't just 'n', because 'n' might not be positive. And
2541/// unfortunately this can come up even for loops where the user didn't use
2542/// a C do-while loop. For example, seemingly well-behaved top-test loops
2543/// will commonly be lowered like this:
2544///
2545/// if (n > 0) {
2546/// i = 0;
2547/// do {
2548/// p[i] = 0.0;
2549/// } while (++i < n);
2550/// }
2551///
2552/// and then it's possible for subsequent optimization to obscure the if
2553/// test in such a way that indvars can't find it.
2554///
2555/// When indvars can't find the if test in loops like this, it creates a
2556/// max expression, which allows it to give the loop a canonical
2557/// induction variable:
2558///
2559/// i = 0;
2560/// max = n < 1 ? 1 : n;
2561/// do {
2562/// p[i] = 0.0;
2563/// } while (++i != max);
2564///
2565/// Canonical induction variables are necessary because the loop passes
2566/// are designed around them. The most obvious example of this is the
2567/// LoopInfo analysis, which doesn't remember trip count values. It
2568/// expects to be able to rediscover the trip count each time it is
2569/// needed, and it does this using a simple analysis that only succeeds if
2570/// the loop has a canonical induction variable.
2571///
2572/// However, when it comes time to generate code, the maximum operation
2573/// can be quite costly, especially if it's inside of an outer loop.
2574///
2575/// This function solves this problem by detecting this type of loop and
2576/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting
2577/// the instructions for the maximum computation.
2578Instruction *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse *&CondUse) {
2579 // Check that the loop matches the pattern we're looking for.
2580 if (Cond->getPredicate() != CmpInst::ICMP_EQ &&
2581 Cond->getPredicate() != CmpInst::ICMP_NE)
2582 return Cond;
2583
2584 SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1));
2585 if (!Sel || !Sel->hasOneUse()) return Cond;
2586
2587 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2588 if (isa<SCEVCouldNotCompute>(BackedgeTakenCount))
2589 return Cond;
2590 const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1);
2591
2592 // Add one to the backedge-taken count to get the trip count.
2593 const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount);
2594 if (IterationCount != SE.getSCEV(Sel)) return Cond;
2595
2596 // Check for a max calculation that matches the pattern. There's no check
2597 // for ICMP_ULE here because the comparison would be with zero, which
2598 // isn't interesting.
2599 CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
2600 const SCEVNAryExpr *Max = nullptr;
2601 if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(BackedgeTakenCount)) {
2602 Pred = ICmpInst::ICMP_SLE;
2603 Max = S;
2604 } else if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(IterationCount)) {
2605 Pred = ICmpInst::ICMP_SLT;
2606 Max = S;
2607 } else if (const SCEVUMaxExpr *U = dyn_cast<SCEVUMaxExpr>(IterationCount)) {
2608 Pred = ICmpInst::ICMP_ULT;
2609 Max = U;
2610 } else {
2611 // No match; bail.
2612 return Cond;
2613 }
2614
2615 // To handle a max with more than two operands, this optimization would
2616 // require additional checking and setup.
2617 if (Max->getNumOperands() != 2)
2618 return Cond;
2619
2620 const SCEV *MaxLHS = Max->getOperand(0);
2621 const SCEV *MaxRHS = Max->getOperand(1);
2622
2623 // ScalarEvolution canonicalizes constants to the left. For < and >, look
2624 // for a comparison with 1. For <= and >=, a comparison with zero.
2625 if (!MaxLHS ||
2626 (ICmpInst::isTrueWhenEqual(Pred) ? !MaxLHS->isZero() : (MaxLHS != One)))
2627 return Cond;
2628
2629 // Check the relevant induction variable for conformance to
2630 // the pattern.
2631 const SCEV *IV = SE.getSCEV(Cond->getOperand(0));
2632 if (!match(IV,
2634 return Cond;
2635
2636 assert(cast<SCEVAddRecExpr>(IV)->getLoop() == L &&
2637 "Loop condition operand is an addrec in a different loop!");
2638
2639 // Check the right operand of the select, and remember it, as it will
2640 // be used in the new comparison instruction.
2641 Value *NewRHS = nullptr;
2642 if (ICmpInst::isTrueWhenEqual(Pred)) {
2643 // Look for n+1, and grab n.
2644 if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(1)))
2645 if (ConstantInt *BO1 = dyn_cast<ConstantInt>(BO->getOperand(1)))
2646 if (BO1->isOne() && SE.getSCEV(BO->getOperand(0)) == MaxRHS)
2647 NewRHS = BO->getOperand(0);
2648 if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(2)))
2649 if (ConstantInt *BO1 = dyn_cast<ConstantInt>(BO->getOperand(1)))
2650 if (BO1->isOne() && SE.getSCEV(BO->getOperand(0)) == MaxRHS)
2651 NewRHS = BO->getOperand(0);
2652 if (!NewRHS)
2653 return Cond;
2654 } else if (SE.getSCEV(Sel->getOperand(1)) == MaxRHS)
2655 NewRHS = Sel->getOperand(1);
2656 else if (SE.getSCEV(Sel->getOperand(2)) == MaxRHS)
2657 NewRHS = Sel->getOperand(2);
2658 else if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(MaxRHS))
2659 NewRHS = SU->getValue();
2660 else
2661 // Max doesn't match expected pattern.
2662 return Cond;
2663
2664 // Determine the new comparison opcode. It may be signed or unsigned,
2665 // and the original comparison may be either equality or inequality.
2666 if (Cond->getPredicate() == CmpInst::ICMP_EQ)
2668
2669 // Ok, everything looks ok to change the condition into an SLT or SGE and
2670 // delete the max calculation.
2671 ICmpInst *NewCond = new ICmpInst(Cond->getIterator(), Pred,
2672 Cond->getOperand(0), NewRHS, "scmp");
2673
2674 // Delete the max calculation instructions.
2675 NewCond->setDebugLoc(Cond->getDebugLoc());
2676 Cond->replaceAllUsesWith(NewCond);
2677 CondUse->setUser(NewCond);
2679 Cond->eraseFromParent();
2680 Sel->eraseFromParent();
2681 if (Cmp->use_empty()) {
2682 salvageDebugInfo(*Cmp);
2683 Cmp->eraseFromParent();
2684 }
2685 return NewCond;
2686}
2687
2688/// Change loop terminating condition to use the postinc iv when possible.
2689void
2690LSRInstance::OptimizeLoopTermCond() {
2691 SmallPtrSet<Instruction *, 4> PostIncs;
2692
2693 // We need a different set of heuristics for rotated and non-rotated loops.
2694 // If a loop is rotated then the latch is also the backedge, so inserting
2695 // post-inc expressions just before the latch is ideal. To reduce live ranges
2696 // it also makes sense to rewrite terminating conditions to use post-inc
2697 // expressions.
2698 //
2699 // If the loop is not rotated then the latch is not a backedge; the latch
2700 // check is done in the loop head. Adding post-inc expressions before the
2701 // latch will cause overlapping live-ranges of pre-inc and post-inc expressions
2702 // in the loop body. In this case we do *not* want to use post-inc expressions
2703 // in the latch check, and we want to insert post-inc expressions before
2704 // the backedge.
2705 BasicBlock *LatchBlock = L->getLoopLatch();
2706 SmallVector<BasicBlock*, 8> ExitingBlocks;
2707 L->getExitingBlocks(ExitingBlocks);
2708 if (!llvm::is_contained(ExitingBlocks, LatchBlock)) {
2709 // The backedge doesn't exit the loop; treat this as a head-tested loop.
2710 IVIncInsertPos = LatchBlock->getTerminator();
2711 return;
2712 }
2713
2714 // Otherwise treat this as a rotated loop.
2715 for (BasicBlock *ExitingBlock : ExitingBlocks) {
2716 // Get the terminating condition for the loop if possible. If we
2717 // can, we want to change it to use a post-incremented version of its
2718 // induction variable, to allow coalescing the live ranges for the IV into
2719 // one register value.
2720
2721 CondBrInst *TermBr = dyn_cast<CondBrInst>(ExitingBlock->getTerminator());
2722 if (!TermBr)
2723 continue;
2724
2726 // If the argument to TermBr is an extractelement, then the source of that
2727 // instruction is what's generated the condition.
2729 if (Extract)
2730 Cond = dyn_cast<Instruction>(Extract->getVectorOperand());
2731 // FIXME: We could do more here, like handling logical operations where one
2732 // side is a cmp that uses an induction variable.
2733 if (!Cond)
2734 continue;
2735
2736 // Search IVUsesByStride to find Cond's IVUse if there is one.
2737 IVStrideUse *CondUse = nullptr;
2738 if (!FindIVUserForCond(Cond, CondUse))
2739 continue;
2740
2741 // If the trip count is computed in terms of a max (due to ScalarEvolution
2742 // being unable to find a sufficient guard, for example), change the loop
2743 // comparison to use SLT or ULT instead of NE.
2744 // One consequence of doing this now is that it disrupts the count-down
2745 // optimization. That's not always a bad thing though, because in such
2746 // cases it may still be worthwhile to avoid a max.
2747 if (auto *Cmp = dyn_cast<ICmpInst>(Cond))
2748 Cond = OptimizeMax(Cmp, CondUse);
2749
2750 // If this exiting block dominates the latch block, it may also use
2751 // the post-inc value if it won't be shared with other uses.
2752 // Check for dominance.
2753 if (!DT.dominates(ExitingBlock, LatchBlock))
2754 continue;
2755
2756 if (auto *Cmp = dyn_cast<ICmpInst>(Cond))
2757 Cond = OptimizeLoopTermCondToEquality(Cmp, CondUse, ExitingBlock);
2758
2759 // Conservatively avoid trying to use the post-inc value in non-latch
2760 // exits if there may be pre-inc users in intervening blocks.
2761 if (LatchBlock != ExitingBlock)
2762 for (const IVStrideUse &UI : IU)
2763 // Test if the use is reachable from the exiting block. This dominator
2764 // query is a conservative approximation of reachability.
2765 if (&UI != CondUse &&
2766 !DT.properlyDominates(UI.getUser()->getParent(), ExitingBlock)) {
2767 // Conservatively assume there may be reuse if the quotient of their
2768 // strides could be a legal scale.
2769 const SCEV *A = IU.getStride(*CondUse, L);
2770 const SCEV *B = IU.getStride(UI, L);
2771 if (!A || !B) continue;
2772 if (SE.getTypeSizeInBits(A->getType()) !=
2773 SE.getTypeSizeInBits(B->getType())) {
2774 if (SE.getTypeSizeInBits(A->getType()) >
2775 SE.getTypeSizeInBits(B->getType()))
2776 B = SE.getSignExtendExpr(B, A->getType());
2777 else
2778 A = SE.getSignExtendExpr(A, B->getType());
2779 }
2780 if (const SCEVConstant *D =
2782 const ConstantInt *C = D->getValue();
2783 // Stride of one or negative one can have reuse with non-addresses.
2784 if (C->isOne() || C->isMinusOne())
2785 goto decline_post_inc;
2786 // Avoid weird situations.
2787 if (C->getValue().getSignificantBits() >= 64 ||
2788 C->getValue().isMinSignedValue())
2789 goto decline_post_inc;
2790 // Check for possible scaled-address reuse.
2791 if (isAddressUse(TTI, UI.getUser(), UI.getOperandValToReplace())) {
2792 MemAccessTy AccessTy =
2793 getAccessType(TTI, UI.getUser(), UI.getOperandValToReplace());
2794 int64_t Scale = C->getSExtValue();
2795 if (TTI.isLegalAddressingMode(AccessTy.MemTy, /*BaseGV=*/nullptr,
2796 /*BaseOffset=*/0,
2797 /*HasBaseReg=*/true, Scale,
2798 AccessTy.AddrSpace))
2799 goto decline_post_inc;
2800 Scale = -Scale;
2801 if (TTI.isLegalAddressingMode(AccessTy.MemTy, /*BaseGV=*/nullptr,
2802 /*BaseOffset=*/0,
2803 /*HasBaseReg=*/true, Scale,
2804 AccessTy.AddrSpace))
2805 goto decline_post_inc;
2806 }
2807 }
2808 }
2809
2810 LLVM_DEBUG(dbgs() << " Change loop exiting icmp to use postinc iv: "
2811 << *Cond << '\n');
2812
2813 // It's possible for the setcc instruction to be anywhere in the loop, and
2814 // possible for it to have multiple users. If it is not immediately before
2815 // the exiting block branch, move it.
2816 if (isa_and_nonnull<CmpInst>(Cond) && Cond->getNextNode() != TermBr &&
2817 !Extract) {
2818 if (Cond->hasOneUse()) {
2819 Cond->moveBefore(TermBr->getIterator());
2820 } else {
2821 // Clone the terminating condition and insert into the loopend.
2822 Instruction *OldCond = Cond;
2823 Cond = Cond->clone();
2824 Cond->setName(L->getHeader()->getName() + ".termcond");
2825 Cond->insertInto(ExitingBlock, TermBr->getIterator());
2826
2827 // Clone the IVUse, as the old use still exists!
2828 CondUse = &IU.AddUser(Cond, CondUse->getOperandValToReplace());
2829 TermBr->replaceUsesOfWith(OldCond, Cond);
2830 }
2831 }
2832
2833 // If we get to here, we know that we can transform the setcc instruction to
2834 // use the post-incremented version of the IV, allowing us to coalesce the
2835 // live ranges for the IV correctly.
2836 CondUse->transformToPostInc(L);
2837 Changed = true;
2838
2839 PostIncs.insert(Cond);
2840 decline_post_inc:;
2841 }
2842
2843 // Determine an insertion point for the loop induction variable increment. It
2844 // must dominate all the post-inc comparisons we just set up, and it must
2845 // dominate the loop latch edge.
2846 IVIncInsertPos = L->getLoopLatch()->getTerminator();
2847 for (Instruction *Inst : PostIncs)
2848 IVIncInsertPos = DT.findNearestCommonDominator(IVIncInsertPos, Inst);
2849}
2850
2851/// Determine if the given use can accommodate a fixup at the given offset and
2852/// other details. If so, update the use and return true.
2853bool LSRInstance::reconcileNewOffset(LSRUse &LU, Immediate NewOffset,
2854 bool HasBaseReg, LSRUse::KindType Kind,
2855 MemAccessTy AccessTy) {
2856 Immediate NewMinOffset = LU.MinOffset;
2857 Immediate NewMaxOffset = LU.MaxOffset;
2858 MemAccessTy NewAccessTy = AccessTy;
2859
2860 // Check for a mismatched kind. It's tempting to collapse mismatched kinds to
2861 // something conservative, however this can pessimize in the case that one of
2862 // the uses will have all its uses outside the loop, for example.
2863 if (LU.Kind != Kind)
2864 return false;
2865
2866 // Check for a mismatched access type, and fall back conservatively as needed.
2867 // TODO: Be less conservative when the type is similar and can use the same
2868 // addressing modes.
2869 if (Kind == LSRUse::Address) {
2870 if (AccessTy.MemTy != LU.AccessTy.MemTy) {
2871 NewAccessTy = MemAccessTy::getUnknown(AccessTy.MemTy->getContext(),
2872 AccessTy.AddrSpace);
2873 }
2874 }
2875
2876 // Conservatively assume HasBaseReg is true for now.
2877 if (Immediate::isKnownLT(NewOffset, LU.MinOffset)) {
2878 if (!isAlwaysFoldable(Opts, TTI, Kind, NewAccessTy, /*BaseGV=*/nullptr,
2879 LU.MaxOffset - NewOffset, HasBaseReg))
2880 return false;
2881 NewMinOffset = NewOffset;
2882 } else if (Immediate::isKnownGT(NewOffset, LU.MaxOffset)) {
2883 if (!isAlwaysFoldable(Opts, TTI, Kind, NewAccessTy, /*BaseGV=*/nullptr,
2884 NewOffset - LU.MinOffset, HasBaseReg))
2885 return false;
2886 NewMaxOffset = NewOffset;
2887 }
2888
2889 // FIXME: We should be able to handle some level of scalable offset support
2890 // for 'void', but in order to get basic support up and running this is
2891 // being left out.
2892 if (NewAccessTy.MemTy && NewAccessTy.MemTy->isVoidTy() &&
2893 (NewMinOffset.isScalable() || NewMaxOffset.isScalable()))
2894 return false;
2895
2896 // Update the use.
2897 LU.MinOffset = NewMinOffset;
2898 LU.MaxOffset = NewMaxOffset;
2899 LU.AccessTy = NewAccessTy;
2900 return true;
2901}
2902
2903/// Return an LSRUse index and an offset value for a fixup which needs the given
2904/// expression, with the given kind and optional access type. Either reuse an
2905/// existing use or create a new one, as needed.
2906std::pair<size_t, Immediate> LSRInstance::getUse(const SCEV *&Expr,
2907 LSRUse::KindType Kind,
2908 MemAccessTy AccessTy) {
2909 const SCEV *Copy = Expr;
2910 SCEVUse ExprUse = Expr;
2911 Immediate Offset = extractImmediate(
2912 Opts, ExprUse, SE, AccessTy.MemTy && AccessTy.MemTy->isScalableTy());
2913 Expr = ExprUse;
2914
2915 // Basic uses can't accept any offset, for example.
2916 if (!isAlwaysFoldable(Opts, TTI, Kind, AccessTy, /*BaseGV=*/nullptr, Offset,
2917 /*HasBaseReg=*/true)) {
2918 Expr = Copy;
2919 Offset = Immediate::getFixed(0);
2920 }
2921
2922 std::pair<UseMapTy::iterator, bool> P =
2923 UseMap.try_emplace(LSRUse::SCEVUseKindPair(Expr, Kind));
2924 if (!P.second) {
2925 // A use already existed with this base.
2926 size_t LUIdx = P.first->second;
2927 LSRUse &LU = Uses[LUIdx];
2928 if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy))
2929 // Reuse this use.
2930 return std::make_pair(LUIdx, Offset);
2931 }
2932
2933 // Create a new use.
2934 size_t LUIdx = Uses.size();
2935 P.first->second = LUIdx;
2936 Uses.push_back(LSRUse(Kind, AccessTy));
2937 LSRUse &LU = Uses[LUIdx];
2938
2939 LU.MinOffset = Offset;
2940 LU.MaxOffset = Offset;
2941 return std::make_pair(LUIdx, Offset);
2942}
2943
2944/// Delete the given use from the Uses list.
2945void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) {
2946 if (&LU != &Uses.back())
2947 std::swap(LU, Uses.back());
2948 Uses.pop_back();
2949
2950 // Update RegUses.
2951 RegUses.swapAndDropUse(LUIdx, Uses.size());
2952}
2953
2954/// Look for a use distinct from OrigLU which is has a formula that has the same
2955/// registers as the given formula.
2956LSRUse *
2957LSRInstance::FindUseWithSimilarFormula(const Formula &OrigF,
2958 const LSRUse &OrigLU) {
2959 // Search all uses for the formula. This could be more clever.
2960 for (LSRUse &LU : Uses) {
2961 // Check whether this use is close enough to OrigLU, to see whether it's
2962 // worthwhile looking through its formulae.
2963 // Ignore ICmpZero uses because they may contain formulae generated by
2964 // GenerateICmpZeroScales, in which case adding fixup offsets may
2965 // be invalid.
2966 if (&LU != &OrigLU && LU.Kind != LSRUse::ICmpZero &&
2967 LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy &&
2968 LU.HasFormulaWithSameRegs(OrigF)) {
2969 // Scan through this use's formulae.
2970 for (const Formula &F : LU.Formulae) {
2971 // Check to see if this formula has the same registers and symbols
2972 // as OrigF.
2973 if (F.BaseRegs == OrigF.BaseRegs &&
2974 F.ScaledReg == OrigF.ScaledReg &&
2975 F.BaseGV == OrigF.BaseGV &&
2976 F.Scale == OrigF.Scale &&
2977 F.UnfoldedOffset == OrigF.UnfoldedOffset) {
2978 if (F.BaseOffset.isZero())
2979 return &LU;
2980 // This is the formula where all the registers and symbols matched;
2981 // there aren't going to be any others. Since we declined it, we
2982 // can skip the rest of the formulae and proceed to the next LSRUse.
2983 break;
2984 }
2985 }
2986 }
2987 }
2988
2989 // Nothing looked good.
2990 return nullptr;
2991}
2992
2993void LSRInstance::CollectInterestingTypesAndFactors() {
2994 SmallSetVector<const SCEV *, 4> Strides;
2995
2996 // Collect interesting types and strides.
2998 for (const IVStrideUse &U : IU) {
2999 const SCEV *Expr = IU.getExpr(U);
3000 if (!Expr)
3001 continue;
3002
3003 // Collect interesting types.
3004 Types.insert(SE.getEffectiveSCEVType(Expr->getType()));
3005
3006 // Add strides for mentioned loops.
3007 Worklist.push_back(Expr);
3008 do {
3009 const SCEV *S = Worklist.pop_back_val();
3010 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
3011 if (AR->getLoop() == L)
3012 Strides.insert(AR->getStepRecurrence(SE));
3013 Worklist.push_back(AR->getStart());
3014 } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
3015 append_range(Worklist, Add->operands());
3016 }
3017 } while (!Worklist.empty());
3018 }
3019
3020 // Compute interesting factors from the set of interesting strides.
3021 for (SmallSetVector<const SCEV *, 4>::const_iterator
3022 I = Strides.begin(), E = Strides.end(); I != E; ++I)
3023 for (SmallSetVector<const SCEV *, 4>::const_iterator NewStrideIter =
3024 std::next(I); NewStrideIter != E; ++NewStrideIter) {
3025 const SCEV *OldStride = *I;
3026 const SCEV *NewStride = *NewStrideIter;
3027
3028 if (SE.getTypeSizeInBits(OldStride->getType()) !=
3029 SE.getTypeSizeInBits(NewStride->getType())) {
3030 if (SE.getTypeSizeInBits(OldStride->getType()) >
3031 SE.getTypeSizeInBits(NewStride->getType()))
3032 NewStride = SE.getSignExtendExpr(NewStride, OldStride->getType());
3033 else
3034 OldStride = SE.getSignExtendExpr(OldStride, NewStride->getType());
3035 }
3036 if (const SCEVConstant *Factor =
3037 dyn_cast_or_null<SCEVConstant>(getExactSDiv(NewStride, OldStride,
3038 SE, true))) {
3039 if (Factor->getAPInt().getSignificantBits() <= 64 && !Factor->isZero())
3040 Factors.insert(Factor->getAPInt().getSExtValue());
3041 } else if (const SCEVConstant *Factor =
3043 NewStride,
3044 SE, true))) {
3045 if (Factor->getAPInt().getSignificantBits() <= 64 && !Factor->isZero())
3046 Factors.insert(Factor->getAPInt().getSExtValue());
3047 }
3048 }
3049
3050 // If all uses use the same type, don't bother looking for truncation-based
3051 // reuse.
3052 if (Types.size() == 1)
3053 Types.clear();
3054
3055 LLVM_DEBUG(print_factors_and_types(dbgs()));
3056}
3057
3058/// Helper for CollectChains that finds an IV operand (computed by an AddRec in
3059/// this loop) within [OI,OE) or returns OE. If IVUsers mapped Instructions to
3060/// IVStrideUses, we could partially skip this.
3061static User::op_iterator
3063 Loop *L, ScalarEvolution &SE) {
3064 for(; OI != OE; ++OI) {
3065 if (Instruction *Oper = dyn_cast<Instruction>(*OI)) {
3066 if (!SE.isSCEVable(Oper->getType()))
3067 continue;
3068
3069 if (const SCEVAddRecExpr *AR =
3071 if (AR->getLoop() == L)
3072 break;
3073 }
3074 }
3075 }
3076 return OI;
3077}
3078
3079/// IVChain logic must consistently peek base TruncInst operands, so wrap it in
3080/// a convenient helper.
3082 if (TruncInst *Trunc = dyn_cast<TruncInst>(Oper))
3083 return Trunc->getOperand(0);
3084 return Oper;
3085}
3086
3087/// Return an approximation of this SCEV expression's "base", or NULL for any
3088/// constant. Returning the expression itself is conservative. Returning a
3089/// deeper subexpression is more precise and valid as long as it isn't less
3090/// complex than another subexpression. For expressions involving multiple
3091/// unscaled values, we need to return the pointer-type SCEVUnknown. This avoids
3092/// forming chains across objects, such as: PrevOper==a[i], IVOper==b[i],
3093/// IVInc==b-a.
3094///
3095/// Since SCEVUnknown is the rightmost type, and pointers are the rightmost
3096/// SCEVUnknown, we simply return the rightmost SCEV operand.
3097static const SCEV *getExprBase(const SCEV *S) {
3098 switch (S->getSCEVType()) {
3099 default: // including scUnknown.
3100 return S;
3101 case scConstant:
3102 case scVScale:
3103 return nullptr;
3104 case scTruncate:
3105 return getExprBase(cast<SCEVTruncateExpr>(S)->getOperand());
3106 case scZeroExtend:
3107 return getExprBase(cast<SCEVZeroExtendExpr>(S)->getOperand());
3108 case scSignExtend:
3109 return getExprBase(cast<SCEVSignExtendExpr>(S)->getOperand());
3110 case scAddExpr: {
3111 // Skip over scaled operands (scMulExpr) to follow add operands as long as
3112 // there's nothing more complex.
3113 // FIXME: not sure if we want to recognize negation.
3114 const SCEVAddExpr *Add = cast<SCEVAddExpr>(S);
3115 for (const SCEV *SubExpr : reverse(Add->operands())) {
3116 if (SubExpr->getSCEVType() == scAddExpr)
3117 return getExprBase(SubExpr);
3118
3119 if (SubExpr->getSCEVType() != scMulExpr)
3120 return SubExpr;
3121 }
3122 return S; // all operands are scaled, be conservative.
3123 }
3124 case scAddRecExpr:
3125 return getExprBase(cast<SCEVAddRecExpr>(S)->getStart());
3126 }
3127 llvm_unreachable("Unknown SCEV kind!");
3128}
3129
3130/// Return true if the chain increment is profitable to expand into a loop
3131/// invariant value, which may require its own register. A profitable chain
3132/// increment will be an offset relative to the same base. We allow such offsets
3133/// to potentially be used as chain increment as long as it's not obviously
3134/// expensive to expand using real instructions.
3135bool IVChain::isProfitableIncrement(const SCEV *OperExpr,
3136 const SCEV *IncExpr,
3137 ScalarEvolution &SE) {
3138 // Do not replace a constant offset from IV head with a nonconstant IV
3139 // increment.
3140 if (!isa<SCEVConstant>(IncExpr)) {
3141 const SCEV *HeadExpr = SE.getSCEV(getWideOperand(Incs[0].IVOperand));
3142 if (isa<SCEVConstant>(SE.getMinusSCEV(OperExpr, HeadExpr)))
3143 return false;
3144 }
3145
3146 SmallPtrSet<const SCEV*, 8> Processed;
3147 return !isHighCostExpansion(IncExpr, Processed, SE);
3148}
3149
3150/// Return true if the number of registers needed for the chain is estimated to
3151/// be less than the number required for the individual IV users. First prohibit
3152/// any IV users that keep the IV live across increments (the Users set should
3153/// be empty). Next count the number and type of increments in the chain.
3154///
3155/// Chaining IVs can lead to considerable code bloat if ISEL doesn't
3156/// effectively use postinc addressing modes. Only consider it profitable it the
3157/// increments can be computed in fewer registers when chained.
3158///
3159/// TODO: Consider IVInc free if it's already used in another chains.
3160static bool isProfitableChain(IVChain &Chain,
3162 ScalarEvolution &SE,
3163 const TargetTransformInfo &TTI) {
3164 if (!Chain.hasIncs())
3165 return false;
3166
3167 if (!Users.empty()) {
3168 LLVM_DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " users:\n";
3169 for (Instruction *Inst
3170 : Users) { dbgs() << " " << *Inst << "\n"; });
3171 return false;
3172 }
3173 assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
3174
3175 // The chain itself may require a register, so initialize cost to 1.
3176 int cost = 1;
3177
3178 // A complete chain likely eliminates the need for keeping the original IV in
3179 // a register. LSR does not currently know how to form a complete chain unless
3180 // the header phi already exists.
3181 if (isa<PHINode>(Chain.tailUserInst())
3182 && SE.getSCEV(Chain.tailUserInst()) == Chain.Incs[0].IncExpr) {
3183 --cost;
3184 }
3185 const SCEV *LastIncExpr = nullptr;
3186 unsigned NumConstIncrements = 0;
3187 unsigned NumVarIncrements = 0;
3188 unsigned NumReusedIncrements = 0;
3189
3190 if (TTI.isProfitableLSRChainElement(Chain.Incs[0].UserInst))
3191 return true;
3192
3193 for (const IVInc &Inc : Chain) {
3194 if (TTI.isProfitableLSRChainElement(Inc.UserInst))
3195 return true;
3196 if (Inc.IncExpr->isZero())
3197 continue;
3198
3199 // Incrementing by zero or some constant is neutral. We assume constants can
3200 // be folded into an addressing mode or an add's immediate operand.
3201 if (isa<SCEVConstant>(Inc.IncExpr)) {
3202 ++NumConstIncrements;
3203 continue;
3204 }
3205
3206 if (Inc.IncExpr == LastIncExpr)
3207 ++NumReusedIncrements;
3208 else
3209 ++NumVarIncrements;
3210
3211 LastIncExpr = Inc.IncExpr;
3212 }
3213 // An IV chain with a single increment is handled by LSR's postinc
3214 // uses. However, a chain with multiple increments requires keeping the IV's
3215 // value live longer than it needs to be if chained.
3216 if (NumConstIncrements > 1)
3217 --cost;
3218
3219 // Materializing increment expressions in the preheader that didn't exist in
3220 // the original code may cost a register. For example, sign-extended array
3221 // indices can produce ridiculous increments like this:
3222 // IV + ((sext i32 (2 * %s) to i64) + (-1 * (sext i32 %s to i64)))
3223 cost += NumVarIncrements;
3224
3225 // Reusing variable increments likely saves a register to hold the multiple of
3226 // the stride.
3227 cost -= NumReusedIncrements;
3228
3229 LLVM_DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " Cost: " << cost
3230 << "\n");
3231
3232 return cost < 0;
3233}
3234
3235/// Add this IV user to an existing chain or make it the head of a new chain.
3236void LSRInstance::ChainInstruction(Instruction *UserInst, Instruction *IVOper,
3237 SmallVectorImpl<ChainUsers> &ChainUsersVec) {
3238 // When IVs are used as types of varying widths, they are generally converted
3239 // to a wider type with some uses remaining narrow under a (free) trunc.
3240 Value *const NextIV = getWideOperand(IVOper);
3241 const SCEV *const OperExpr = SE.getSCEV(NextIV);
3242 const SCEV *const OperExprBase = getExprBase(OperExpr);
3243
3244 // Visit all existing chains. Check if its IVOper can be computed as a
3245 // profitable loop invariant increment from the last link in the Chain.
3246 unsigned ChainIdx = 0, NChains = IVChainVec.size();
3247 const SCEV *LastIncExpr = nullptr;
3248 for (; ChainIdx < NChains; ++ChainIdx) {
3249 IVChain &Chain = IVChainVec[ChainIdx];
3250
3251 // Prune the solution space aggressively by checking that both IV operands
3252 // are expressions that operate on the same unscaled SCEVUnknown. This
3253 // "base" will be canceled by the subsequent getMinusSCEV call. Checking
3254 // first avoids creating extra SCEV expressions.
3255 if (!stressIVChain(Opts) && Chain.ExprBase != OperExprBase)
3256 continue;
3257
3258 Value *PrevIV = getWideOperand(Chain.Incs.back().IVOperand);
3259 if (PrevIV->getType() != NextIV->getType())
3260 continue;
3261
3262 // A phi node terminates a chain.
3263 if (isa<PHINode>(UserInst) && isa<PHINode>(Chain.tailUserInst()))
3264 continue;
3265
3266 // The increment must be loop-invariant so it can be kept in a register.
3267 const SCEV *PrevExpr = SE.getSCEV(PrevIV);
3268 const SCEV *IncExpr = SE.getMinusSCEV(OperExpr, PrevExpr);
3269 if (isa<SCEVCouldNotCompute>(IncExpr) || !SE.isLoopInvariant(IncExpr, L))
3270 continue;
3271
3272 // Aggressively form chains when -stress-ivchain.
3273 if (stressIVChain(Opts) ||
3274 Chain.isProfitableIncrement(OperExpr, IncExpr, SE)) {
3275 LastIncExpr = IncExpr;
3276 break;
3277 }
3278 }
3279 // If we haven't found a chain, create a new one, unless we hit the max. Don't
3280 // bother for phi nodes, because they must be last in the chain.
3281 if (ChainIdx == NChains) {
3282 if (isa<PHINode>(UserInst))
3283 return;
3284 if (NChains >= MaxChains && !stressIVChain(Opts)) {
3285 LLVM_DEBUG(dbgs() << "IV Chain Limit\n");
3286 return;
3287 }
3288 LastIncExpr = OperExpr;
3289 // IVUsers may have skipped over sign/zero extensions. We don't currently
3290 // attempt to form chains involving extensions unless they can be hoisted
3291 // into this loop's AddRec.
3292 if (!isa<SCEVAddRecExpr>(LastIncExpr))
3293 return;
3294 ++NChains;
3295 IVChainVec.push_back(IVChain(IVInc(UserInst, IVOper, LastIncExpr),
3296 OperExprBase));
3297 ChainUsersVec.resize(NChains);
3298 LLVM_DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Head: (" << *UserInst
3299 << ") IV=" << *LastIncExpr << "\n");
3300 } else {
3301 LLVM_DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Inc: (" << *UserInst
3302 << ") IV+" << *LastIncExpr << "\n");
3303 // Add this IV user to the end of the chain.
3304 IVChainVec[ChainIdx].add(IVInc(UserInst, IVOper, LastIncExpr));
3305 }
3306 IVChain &Chain = IVChainVec[ChainIdx];
3307
3308 SmallPtrSet<Instruction*,4> &NearUsers = ChainUsersVec[ChainIdx].NearUsers;
3309 // This chain's NearUsers become FarUsers.
3310 if (!LastIncExpr->isZero()) {
3311 ChainUsersVec[ChainIdx].FarUsers.insert_range(NearUsers);
3312 NearUsers.clear();
3313 }
3314
3315 // All other uses of IVOperand become near uses of the chain.
3316 // We currently ignore intermediate values within SCEV expressions, assuming
3317 // they will eventually be used be the current chain, or can be computed
3318 // from one of the chain increments. To be more precise we could
3319 // transitively follow its user and only add leaf IV users to the set.
3320 for (User *U : IVOper->users()) {
3321 Instruction *OtherUse = dyn_cast<Instruction>(U);
3322 if (!OtherUse)
3323 continue;
3324 // Uses in the chain will no longer be uses if the chain is formed.
3325 // Include the head of the chain in this iteration (not Chain.begin()).
3326 IVChain::const_iterator IncIter = Chain.Incs.begin();
3327 IVChain::const_iterator IncEnd = Chain.Incs.end();
3328 for( ; IncIter != IncEnd; ++IncIter) {
3329 if (IncIter->UserInst == OtherUse)
3330 break;
3331 }
3332 if (IncIter != IncEnd)
3333 continue;
3334
3335 if (SE.isSCEVable(OtherUse->getType())
3336 && !isa<SCEVUnknown>(SE.getSCEV(OtherUse))
3337 && IU.isIVUserOrOperand(OtherUse)) {
3338 continue;
3339 }
3340 NearUsers.insert(OtherUse);
3341 }
3342
3343 // Since this user is part of the chain, it's no longer considered a use
3344 // of the chain.
3345 ChainUsersVec[ChainIdx].FarUsers.erase(UserInst);
3346}
3347
3348/// Populate the vector of Chains.
3349///
3350/// This decreases ILP at the architecture level. Targets with ample registers,
3351/// multiple memory ports, and no register renaming probably don't want
3352/// this. However, such targets should probably disable LSR altogether.
3353///
3354/// The job of LSR is to make a reasonable choice of induction variables across
3355/// the loop. Subsequent passes can easily "unchain" computation exposing more
3356/// ILP *within the loop* if the target wants it.
3357///
3358/// Finding the best IV chain is potentially a scheduling problem. Since LSR
3359/// will not reorder memory operations, it will recognize this as a chain, but
3360/// will generate redundant IV increments. Ideally this would be corrected later
3361/// by a smart scheduler:
3362/// = A[i]
3363/// = A[i+x]
3364/// A[i] =
3365/// A[i+x] =
3366///
3367/// TODO: Walk the entire domtree within this loop, not just the path to the
3368/// loop latch. This will discover chains on side paths, but requires
3369/// maintaining multiple copies of the Chains state.
3370void LSRInstance::CollectChains() {
3371 LLVM_DEBUG(dbgs() << "Collecting IV Chains.\n");
3372 SmallVector<ChainUsers, 8> ChainUsersVec;
3373
3374 SmallVector<BasicBlock *,8> LatchPath;
3375 BasicBlock *LoopHeader = L->getHeader();
3376 for (DomTreeNode *Rung = DT.getNode(L->getLoopLatch());
3377 Rung->getBlock() != LoopHeader; Rung = Rung->getIDom()) {
3378 LatchPath.push_back(Rung->getBlock());
3379 }
3380 LatchPath.push_back(LoopHeader);
3381
3382 // Walk the instruction stream from the loop header to the loop latch.
3383 for (BasicBlock *BB : reverse(LatchPath)) {
3384 for (Instruction &I : *BB) {
3385 // Skip instructions that weren't seen by IVUsers analysis.
3386 if (isa<PHINode>(I) || !IU.isIVUserOrOperand(&I))
3387 continue;
3388
3389 // Skip ephemeral values, as they don't produce real code.
3390 if (IU.isEphemeral(&I))
3391 continue;
3392
3393 // Ignore users that are part of a SCEV expression. This way we only
3394 // consider leaf IV Users. This effectively rediscovers a portion of
3395 // IVUsers analysis but in program order this time.
3396 if (SE.isSCEVable(I.getType()) && !isa<SCEVUnknown>(SE.getSCEV(&I)))
3397 continue;
3398
3399 // Remove this instruction from any NearUsers set it may be in.
3400 for (unsigned ChainIdx = 0, NChains = IVChainVec.size();
3401 ChainIdx < NChains; ++ChainIdx) {
3402 ChainUsersVec[ChainIdx].NearUsers.erase(&I);
3403 }
3404 // Search for operands that can be chained.
3405 SmallPtrSet<Instruction*, 4> UniqueOperands;
3406 User::op_iterator IVOpEnd = I.op_end();
3407 User::op_iterator IVOpIter = findIVOperand(I.op_begin(), IVOpEnd, L, SE);
3408 while (IVOpIter != IVOpEnd) {
3409 Instruction *IVOpInst = cast<Instruction>(*IVOpIter);
3410 if (UniqueOperands.insert(IVOpInst).second)
3411 ChainInstruction(&I, IVOpInst, ChainUsersVec);
3412 IVOpIter = findIVOperand(std::next(IVOpIter), IVOpEnd, L, SE);
3413 }
3414 } // Continue walking down the instructions.
3415 } // Continue walking down the domtree.
3416 // Visit phi backedges to determine if the chain can generate the IV postinc.
3417 for (PHINode &PN : L->getHeader()->phis()) {
3418 if (!SE.isSCEVable(PN.getType()))
3419 continue;
3420
3421 Instruction *IncV =
3422 dyn_cast<Instruction>(PN.getIncomingValueForBlock(L->getLoopLatch()));
3423 if (IncV)
3424 ChainInstruction(&PN, IncV, ChainUsersVec);
3425 }
3426 // Remove any unprofitable chains.
3427 unsigned ChainIdx = 0;
3428 for (unsigned UsersIdx = 0, NChains = IVChainVec.size();
3429 UsersIdx < NChains; ++UsersIdx) {
3430 if (!stressIVChain(Opts) &&
3431 !isProfitableChain(IVChainVec[UsersIdx],
3432 ChainUsersVec[UsersIdx].FarUsers, SE, TTI))
3433 continue;
3434 // Preserve the chain at UsesIdx.
3435 if (ChainIdx != UsersIdx)
3436 IVChainVec[ChainIdx] = IVChainVec[UsersIdx];
3437 FinalizeChain(IVChainVec[ChainIdx]);
3438 ++ChainIdx;
3439 }
3440 IVChainVec.resize(ChainIdx);
3441}
3442
3443void LSRInstance::FinalizeChain(IVChain &Chain) {
3444 assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
3445 LLVM_DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[0].UserInst << "\n");
3446
3447 for (const IVInc &Inc : Chain) {
3448 LLVM_DEBUG(dbgs() << " Inc: " << *Inc.UserInst << "\n");
3449 auto UseI = find(Inc.UserInst->operands(), Inc.IVOperand);
3450 assert(UseI != Inc.UserInst->op_end() && "cannot find IV operand");
3451 IVIncSet.insert(UseI);
3452 }
3453}
3454
3455/// Return true if the IVInc can be folded into an addressing mode.
3456static bool canFoldIVIncExpr(const ScalarOptions &Opts, const SCEV *IncExpr,
3457 Instruction *UserInst, Value *Operand,
3458 const TargetTransformInfo &TTI) {
3459 const SCEVConstant *IncConst = dyn_cast<SCEVConstant>(IncExpr);
3460 Immediate IncOffset = Immediate::getZero();
3461 if (IncConst) {
3462 if (IncConst && IncConst->getAPInt().getSignificantBits() > 64)
3463 return false;
3464 IncOffset = Immediate::getFixed(IncConst->getValue()->getSExtValue());
3465 } else {
3466 // Look for mul(vscale, constant), to detect a scalable offset.
3467 const APInt *C;
3468 if (!match(IncExpr, m_scev_Mul(m_scev_APInt(C), m_SCEVVScale())) ||
3469 C->getSignificantBits() > 64)
3470 return false;
3471 IncOffset = Immediate::getScalable(C->getSExtValue());
3472 }
3473
3474 if (!isAddressUse(TTI, UserInst, Operand))
3475 return false;
3476
3477 MemAccessTy AccessTy = getAccessType(TTI, UserInst, Operand);
3478 if (!isAlwaysFoldable(Opts, TTI, LSRUse::Address, AccessTy,
3479 /*BaseGV=*/nullptr, IncOffset, /*HasBaseReg=*/false))
3480 return false;
3481
3482 return true;
3483}
3484
3485/// Generate an add or subtract for each IVInc in a chain to materialize the IV
3486/// user's operand from the previous IV user's operand.
3487void LSRInstance::GenerateIVChain(const IVChain &Chain,
3488 SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
3489 // Find the new IVOperand for the head of the chain. It may have been replaced
3490 // by LSR.
3491 const IVInc &Head = Chain.Incs[0];
3492 User::op_iterator IVOpEnd = Head.UserInst->op_end();
3493 // findIVOperand returns IVOpEnd if it can no longer find a valid IV user.
3494 User::op_iterator IVOpIter = findIVOperand(Head.UserInst->op_begin(),
3495 IVOpEnd, L, SE);
3496 Value *IVSrc = nullptr;
3497 while (IVOpIter != IVOpEnd) {
3498 IVSrc = getWideOperand(*IVOpIter);
3499
3500 // If this operand computes the expression that the chain needs, we may use
3501 // it. (Check this after setting IVSrc which is used below.)
3502 //
3503 // Note that if Head.IncExpr is wider than IVSrc, then this phi is too
3504 // narrow for the chain, so we can no longer use it. We do allow using a
3505 // wider phi, assuming the LSR checked for free truncation. In that case we
3506 // should already have a truncate on this operand such that
3507 // getSCEV(IVSrc) == IncExpr.
3508 if (SE.getSCEV(*IVOpIter) == Head.IncExpr
3509 || SE.getSCEV(IVSrc) == Head.IncExpr) {
3510 break;
3511 }
3512 IVOpIter = findIVOperand(std::next(IVOpIter), IVOpEnd, L, SE);
3513 }
3514 if (IVOpIter == IVOpEnd) {
3515 // Gracefully give up on this chain.
3516 LLVM_DEBUG(dbgs() << "Concealed chain head: " << *Head.UserInst << "\n");
3517 return;
3518 }
3519 assert(IVSrc && "Failed to find IV chain source");
3520
3521 LLVM_DEBUG(dbgs() << "Generate chain at: " << *IVSrc << "\n");
3522 Type *IVTy = IVSrc->getType();
3523 Type *IntTy = SE.getEffectiveSCEVType(IVTy);
3524 const SCEV *LeftOverExpr = nullptr;
3525 const SCEV *Accum = SE.getZero(IntTy);
3527 Bases.emplace_back(Accum, IVSrc);
3528
3529 for (const IVInc &Inc : Chain) {
3530 Instruction *InsertPt = Inc.UserInst;
3532 InsertPt = L->getLoopLatch()->getTerminator();
3533
3534 // IVOper will replace the current IV User's operand. IVSrc is the IV
3535 // value currently held in a register.
3536 Value *IVOper = IVSrc;
3537 if (!Inc.IncExpr->isZero()) {
3538 // IncExpr was the result of subtraction of two narrow values, so must
3539 // be signed.
3540 const SCEV *IncExpr = SE.getNoopOrSignExtend(Inc.IncExpr, IntTy);
3541 Accum = SE.getAddExpr(Accum, IncExpr);
3542 LeftOverExpr = LeftOverExpr
3543 ? SE.getAddExpr(LeftOverExpr, IncExpr).getPointer()
3544 : IncExpr;
3545 }
3546
3547 // Look through each base to see if any can produce a nice addressing mode.
3548 bool FoundBase = false;
3549 for (auto [MapScev, MapIVOper] : reverse(Bases)) {
3550 const SCEV *Remainder = SE.getMinusSCEV(Accum, MapScev);
3551 if (canFoldIVIncExpr(Opts, Remainder, Inc.UserInst, Inc.IVOperand, TTI)) {
3552 if (!Remainder->isZero()) {
3553 Rewriter.clearPostInc();
3554 Value *IncV = Rewriter.expandCodeFor(Remainder, IntTy, InsertPt);
3555 const SCEV *IVOperExpr =
3556 SE.getAddExpr(SE.getUnknown(MapIVOper), SE.getUnknown(IncV));
3557 IVOper = Rewriter.expandCodeFor(IVOperExpr, IVTy, InsertPt);
3558 } else {
3559 IVOper = MapIVOper;
3560 }
3561
3562 FoundBase = true;
3563 break;
3564 }
3565 }
3566 if (!FoundBase && LeftOverExpr && !LeftOverExpr->isZero()) {
3567 // Expand the IV increment.
3568 Rewriter.clearPostInc();
3569 Value *IncV = Rewriter.expandCodeFor(LeftOverExpr, IntTy, InsertPt);
3570 const SCEV *IVOperExpr = SE.getAddExpr(SE.getUnknown(IVSrc),
3571 SE.getUnknown(IncV));
3572 IVOper = Rewriter.expandCodeFor(IVOperExpr, IVTy, InsertPt);
3573
3574 // If an IV increment can't be folded, use it as the next IV value.
3575 if (!canFoldIVIncExpr(Opts, LeftOverExpr, Inc.UserInst, Inc.IVOperand,
3576 TTI)) {
3577 assert(IVTy == IVOper->getType() && "inconsistent IV increment type");
3578 Bases.emplace_back(Accum, IVOper);
3579 IVSrc = IVOper;
3580 LeftOverExpr = nullptr;
3581 }
3582 }
3583 Type *OperTy = Inc.IVOperand->getType();
3584 if (IVTy != OperTy) {
3585 assert(SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) &&
3586 "cannot extend a chained IV");
3587 IRBuilder<> Builder(InsertPt);
3588 IVOper = Builder.CreateTruncOrBitCast(IVOper, OperTy, "lsr.chain");
3589 }
3590 Inc.UserInst->replaceUsesOfWith(Inc.IVOperand, IVOper);
3591 if (auto *OperandIsInstr = dyn_cast<Instruction>(Inc.IVOperand))
3592 DeadInsts.emplace_back(OperandIsInstr);
3593 }
3594 // If LSR created a new, wider phi, we may also replace its postinc. We only
3595 // do this if we also found a wide value for the head of the chain.
3596 if (isa<PHINode>(Chain.tailUserInst())) {
3597 for (PHINode &Phi : L->getHeader()->phis()) {
3598 if (Phi.getType() != IVSrc->getType())
3599 continue;
3601 Phi.getIncomingValueForBlock(L->getLoopLatch()));
3602 if (!PostIncV || (SE.getSCEV(PostIncV) != SE.getSCEV(IVSrc)))
3603 continue;
3604 Value *IVOper = IVSrc;
3605 Type *PostIncTy = PostIncV->getType();
3606 if (IVTy != PostIncTy) {
3607 assert(PostIncTy->isPointerTy() && "mixing int/ptr IV types");
3608 IRBuilder<> Builder(L->getLoopLatch()->getTerminator());
3609 Builder.SetCurrentDebugLocation(PostIncV->getDebugLoc());
3610 IVOper = Builder.CreatePointerCast(IVSrc, PostIncTy, "lsr.chain");
3611 }
3612 Phi.replaceUsesOfWith(PostIncV, IVOper);
3613 DeadInsts.emplace_back(PostIncV);
3614 }
3615 }
3616}
3617
3618void LSRInstance::CollectFixupsAndInitialFormulae() {
3619 CondBrInst *ExitBranch = nullptr;
3620 bool SaveCmp = TTI.canSaveCmp(L, &ExitBranch, &SE, &LI, &DT, &AC, &TLI);
3621
3622 // For calculating baseline cost
3623 SmallPtrSet<const SCEV *, 16> Regs;
3624 DenseSet<const SCEV *> VisitedRegs;
3625 DenseSet<size_t> VisitedLSRUse;
3626
3627 for (const IVStrideUse &U : IU) {
3628 Instruction *UserInst = U.getUser();
3629 // Skip IV users that are part of profitable IV Chains.
3630 User::op_iterator UseI =
3631 find(UserInst->operands(), U.getOperandValToReplace());
3632 assert(UseI != UserInst->op_end() && "cannot find IV operand");
3633 if (IVIncSet.count(UseI)) {
3634 LLVM_DEBUG(dbgs() << "Use is in profitable chain: " << **UseI << '\n');
3635 continue;
3636 }
3637
3638 LSRUse::KindType Kind = LSRUse::Basic;
3639 MemAccessTy AccessTy;
3640 if (isAddressUse(TTI, UserInst, U.getOperandValToReplace())) {
3641 Kind = LSRUse::Address;
3642 AccessTy = getAccessType(TTI, UserInst, U.getOperandValToReplace());
3643 }
3644
3645 const SCEV *S = IU.getExpr(U);
3646 if (!S)
3647 continue;
3648 PostIncLoopSet TmpPostIncLoops = U.getPostIncLoops();
3649
3650 // Equality (== and !=) ICmps are special. We can rewrite (i == N) as
3651 // (N - i == 0), and this allows (N - i) to be the expression that we work
3652 // with rather than just N or i, so we can consider the register
3653 // requirements for both N and i at the same time. Limiting this code to
3654 // equality icmps is not a problem because all interesting loops use
3655 // equality icmps, thanks to IndVarSimplify.
3656 if (ICmpInst *CI = dyn_cast<ICmpInst>(UserInst)) {
3657 // If CI can be saved in some target, like replaced inside hardware loop
3658 // in PowerPC, no need to generate initial formulae for it.
3659 if (SaveCmp && CI == dyn_cast<ICmpInst>(ExitBranch->getCondition()))
3660 continue;
3661 if (CI->isEquality()) {
3662 // Swap the operands if needed to put the OperandValToReplace on the
3663 // left, for consistency.
3664 Value *NV = CI->getOperand(1);
3665 if (NV == U.getOperandValToReplace()) {
3666 CI->setOperand(1, CI->getOperand(0));
3667 CI->setOperand(0, NV);
3668 NV = CI->getOperand(1);
3669 Changed = true;
3670 }
3671
3672 // x == y --> x - y == 0
3673 const SCEV *N = SE.getSCEV(NV);
3674 if (SE.isLoopInvariant(N, L) && Rewriter.isSafeToExpand(N) &&
3675 (!NV->getType()->isPointerTy() ||
3676 SE.getPointerBase(N) == SE.getPointerBase(S))) {
3677 // S is normalized, so normalize N before folding it into S
3678 // to keep the result normalized.
3679 N = normalizeForPostIncUse(N, TmpPostIncLoops, SE);
3680 if (!N)
3681 continue;
3682 Kind = LSRUse::ICmpZero;
3683 S = SE.getMinusSCEV(N, S);
3684 } else if (L->isLoopInvariant(NV) &&
3685 (!isa<Instruction>(NV) ||
3686 DT.dominates(cast<Instruction>(NV), L->getHeader())) &&
3687 !NV->getType()->isPointerTy()) {
3688 // If we can't generally expand the expression (e.g. it contains
3689 // a divide), but it is already at a loop invariant point before the
3690 // loop, wrap it in an unknown (to prevent the expander from trying
3691 // to re-expand in a potentially unsafe way.) The restriction to
3692 // integer types is required because the unknown hides the base, and
3693 // SCEV can't compute the difference of two unknown pointers.
3694 N = SE.getUnknown(NV);
3695 N = normalizeForPostIncUse(N, TmpPostIncLoops, SE);
3696 if (!N)
3697 continue;
3698 Kind = LSRUse::ICmpZero;
3699 S = SE.getMinusSCEV(N, S);
3701 }
3702
3703 // -1 and the negations of all interesting strides (except the negation
3704 // of -1) are now also interesting.
3705 for (size_t i = 0, e = Factors.size(); i != e; ++i)
3706 if (Factors[i] != -1)
3707 Factors.insert(-(uint64_t)Factors[i]);
3708 Factors.insert(-1);
3709 }
3710 }
3711
3712 // Get or create an LSRUse.
3713 std::pair<size_t, Immediate> P = getUse(S, Kind, AccessTy);
3714 size_t LUIdx = P.first;
3715 Immediate Offset = P.second;
3716 LSRUse &LU = Uses[LUIdx];
3717
3718 // Record the fixup.
3719 LSRFixup &LF = LU.getNewFixup();
3720 LF.UserInst = UserInst;
3721 LF.OperandValToReplace = U.getOperandValToReplace();
3722 LF.PostIncLoops = TmpPostIncLoops;
3723 LF.Offset = Offset;
3724 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3725 LU.AllFixupsUnconditional &= IsFixupExecutedEachIncrement(LF);
3726
3727 // Create SCEV as Formula for calculating baseline cost
3728 if (!VisitedLSRUse.count(LUIdx) && !LF.isUseFullyOutsideLoop(L)) {
3729 Formula F;
3730 F.initialMatch(S, L, SE);
3731 BaselineCost.RateFormula(F, Regs, VisitedRegs, LU,
3732 HardwareLoopProfitable);
3733 VisitedLSRUse.insert(LUIdx);
3734 }
3735
3736 // If this is the first use of this LSRUse, give it a formula.
3737 if (LU.Formulae.empty()) {
3738 InsertInitialFormula(S, LU, LUIdx);
3739 CountRegisters(LU.Formulae.back(), LUIdx);
3740 }
3741 }
3742
3743 LLVM_DEBUG(print_fixups(dbgs()));
3744}
3745
3746/// Insert a formula for the given expression into the given use, separating out
3747/// loop-variant portions from loop-invariant and loop-computable portions.
3748void LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU,
3749 size_t LUIdx) {
3750 // Mark uses whose expressions cannot be expanded.
3751 if (!Rewriter.isSafeToExpand(S))
3752 LU.RigidFormula = true;
3753
3754 Formula F;
3755 F.initialMatch(S, L, SE);
3756 bool Inserted = InsertFormula(LU, LUIdx, F);
3757 assert(Inserted && "Initial formula already exists!"); (void)Inserted;
3758}
3759
3760/// Insert a simple single-register formula for the given expression into the
3761/// given use.
3762void
3763LSRInstance::InsertSupplementalFormula(const SCEV *S,
3764 LSRUse &LU, size_t LUIdx) {
3765 Formula F;
3766 F.BaseRegs.push_back(S);
3767 F.HasBaseReg = true;
3768 bool Inserted = InsertFormula(LU, LUIdx, F);
3769 assert(Inserted && "Supplemental formula already exists!"); (void)Inserted;
3770}
3771
3772/// Note which registers are used by the given formula, updating RegUses.
3773void LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) {
3774 if (F.ScaledReg)
3775 RegUses.countRegister(F.ScaledReg, LUIdx);
3776 for (const SCEV *BaseReg : F.BaseRegs)
3777 RegUses.countRegister(BaseReg, LUIdx);
3778}
3779
3780/// If the given formula has not yet been inserted, add it to the list, and
3781/// return true. Return false otherwise.
3782bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) {
3783 // Do not insert formula that we will not be able to expand.
3784 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&
3785 "Formula is illegal");
3786
3787 if (!LU.InsertFormula(F, *L))
3788 return false;
3789
3790 CountRegisters(F, LUIdx);
3791 return true;
3792}
3793
3794/// Test whether this fixup will be executed each time the corresponding IV
3795/// increment instruction is executed.
3796bool LSRInstance::IsFixupExecutedEachIncrement(const LSRFixup &LF) const {
3797 // If the fixup block dominates the IV increment block then there is no path
3798 // through the loop to the increment that doesn't pass through the fixup.
3799 return DT.dominates(LF.UserInst->getParent(), IVIncInsertPos->getParent());
3800}
3801
3802/// Check for other uses of loop-invariant values which we're tracking. These
3803/// other uses will pin these values in registers, making them less profitable
3804/// for elimination.
3805/// TODO: This currently misses non-constant addrec step registers.
3806/// TODO: Should this give more weight to users inside the loop?
3807void
3808LSRInstance::CollectLoopInvariantFixupsAndFormulae() {
3809 SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end());
3810 SmallPtrSet<const SCEV *, 32> Visited;
3811
3812 // Don't collect outside uses if we are favoring postinc - the instructions in
3813 // the loop are more important than the ones outside of it.
3814 if (AMK == TTI::AMK_PostIndexed)
3815 return;
3816
3817 while (!Worklist.empty()) {
3818 const SCEV *S = Worklist.pop_back_val();
3819
3820 // Don't process the same SCEV twice
3821 if (!Visited.insert(S).second)
3822 continue;
3823
3824 if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(S))
3825 append_range(Worklist, N->operands());
3826 else if (const SCEVIntegralCastExpr *C = dyn_cast<SCEVIntegralCastExpr>(S))
3827 Worklist.push_back(C->getOperand());
3828 else if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) {
3829 Worklist.push_back(D->getLHS());
3830 Worklist.push_back(D->getRHS());
3831 } else if (const SCEVUnknown *US = dyn_cast<SCEVUnknown>(S)) {
3832 const Value *V = US->getValue();
3833 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
3834 // Look for instructions defined outside the loop.
3835 if (L->contains(Inst)) continue;
3836 } else if (isa<Constant>(V))
3837 // Constants can be re-materialized.
3838 continue;
3839 for (const Use &U : V->uses()) {
3840 const Instruction *UserInst = dyn_cast<Instruction>(U.getUser());
3841 // Ignore non-instructions.
3842 if (!UserInst)
3843 continue;
3844 // Don't bother if the instruction is an EHPad.
3845 if (UserInst->isEHPad())
3846 continue;
3847 // Ignore instructions in other functions (as can happen with
3848 // Constants).
3849 if (UserInst->getParent()->getParent() != L->getHeader()->getParent())
3850 continue;
3851 // Ignore instructions not dominated by the loop.
3852 const BasicBlock *UseBB = !isa<PHINode>(UserInst) ?
3853 UserInst->getParent() :
3854 cast<PHINode>(UserInst)->getIncomingBlock(
3856 if (!DT.dominates(L->getHeader(), UseBB))
3857 continue;
3858 // Don't bother if the instruction is in a BB which ends in an EHPad.
3859 if (UseBB->getTerminator()->isEHPad())
3860 continue;
3861
3862 // Ignore cases in which the currently-examined value could come from
3863 // a basic block terminated with an EHPad. This checks all incoming
3864 // blocks of the phi node since it is possible that the same incoming
3865 // value comes from multiple basic blocks, only some of which may end
3866 // in an EHPad. If any of them do, a subsequent rewrite attempt by this
3867 // pass would try to insert instructions into an EHPad, hitting an
3868 // assertion.
3869 if (isa<PHINode>(UserInst)) {
3870 const auto *PhiNode = cast<PHINode>(UserInst);
3871 bool HasIncompatibleEHPTerminatedBlock = false;
3872 llvm::Value *ExpectedValue = U;
3873 for (unsigned int I = 0; I < PhiNode->getNumIncomingValues(); I++) {
3874 if (PhiNode->getIncomingValue(I) == ExpectedValue) {
3875 if (PhiNode->getIncomingBlock(I)->getTerminator()->isEHPad()) {
3876 HasIncompatibleEHPTerminatedBlock = true;
3877 break;
3878 }
3879 }
3880 }
3881 if (HasIncompatibleEHPTerminatedBlock) {
3882 continue;
3883 }
3884 }
3885
3886 // Don't bother rewriting PHIs in catchswitch blocks.
3887 if (isa<CatchSwitchInst>(UserInst->getParent()->getTerminator()))
3888 continue;
3889 // Ignore uses which are part of other SCEV expressions, to avoid
3890 // analyzing them multiple times.
3891 if (SE.isSCEVable(UserInst->getType())) {
3892 const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst));
3893 // If the user is a no-op, look through to its uses.
3894 if (!isa<SCEVUnknown>(UserS))
3895 continue;
3896 if (UserS == US) {
3897 Worklist.push_back(
3898 SE.getUnknown(const_cast<Instruction *>(UserInst)));
3899 continue;
3900 }
3901 }
3902 // Ignore icmp instructions which are already being analyzed.
3903 if (const ICmpInst *ICI = dyn_cast<ICmpInst>(UserInst)) {
3904 unsigned OtherIdx = !U.getOperandNo();
3905 Value *OtherOp = ICI->getOperand(OtherIdx);
3906 if (SE.hasComputableLoopEvolution(SE.getSCEV(OtherOp), L))
3907 continue;
3908 }
3909
3910 // Do not consider uses inside lifetime intrinsics. These are not
3911 // actually materialized.
3912 if (UserInst->isLifetimeStartOrEnd())
3913 continue;
3914
3915 std::pair<size_t, Immediate> P =
3916 getUse(S, LSRUse::Basic, MemAccessTy());
3917 size_t LUIdx = P.first;
3918 Immediate Offset = P.second;
3919 LSRUse &LU = Uses[LUIdx];
3920 LSRFixup &LF = LU.getNewFixup();
3921 LF.UserInst = const_cast<Instruction *>(UserInst);
3922 LF.OperandValToReplace = U;
3923 LF.Offset = Offset;
3924 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3925 LU.AllFixupsUnconditional &= IsFixupExecutedEachIncrement(LF);
3926 InsertSupplementalFormula(US, LU, LUIdx);
3927 CountRegisters(LU.Formulae.back(), Uses.size() - 1);
3928 break;
3929 }
3930 }
3931 }
3932}
3933
3934/// Split S into subexpressions which can be pulled out into separate
3935/// registers. If C is non-null, multiply each subexpression by C.
3936///
3937/// Return remainder expression after factoring the subexpressions captured by
3938/// Ops. If Ops is complete, return NULL.
3939static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
3941 const Loop *L,
3942 ScalarEvolution &SE,
3943 unsigned Depth = 0) {
3944 // Arbitrarily cap recursion to protect compile time.
3945 if (Depth >= 3)
3946 return S;
3947
3948 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
3949 // Break out add operands.
3950 for (const SCEV *S : Add->operands()) {
3951 const SCEV *Remainder = CollectSubexprs(S, C, Ops, L, SE, Depth+1);
3952 if (Remainder)
3953 Ops.push_back(C ? SE.getMulExpr(C, Remainder).getPointer() : Remainder);
3954 }
3955 return nullptr;
3956 }
3957 const SCEV *Start, *Step;
3958 const SCEVConstant *Op0;
3959 const SCEV *Op1;
3960 if (match(S, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step)))) {
3961 // Split a non-zero base out of an addrec.
3962 if (Start->isZero())
3963 return S;
3964
3965 const SCEV *Remainder = CollectSubexprs(Start, C, Ops, L, SE, Depth + 1);
3966 // Split the non-zero AddRec unless it is part of a nested recurrence that
3967 // does not pertain to this loop.
3968 if (Remainder && (cast<SCEVAddRecExpr>(S)->getLoop() == L ||
3969 !isa<SCEVAddRecExpr>(Remainder))) {
3970 Ops.push_back(C ? SE.getMulExpr(C, Remainder).getPointer() : Remainder);
3971 Remainder = nullptr;
3972 }
3973 if (Remainder != Start) {
3974 if (!Remainder)
3975 Remainder = SE.getConstant(S->getType(), 0);
3976 return SE.getAddRecExpr(Remainder, Step,
3977 cast<SCEVAddRecExpr>(S)->getLoop(),
3978 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
3980 }
3981 } else if (match(S, m_scev_Mul(m_SCEVConstant(Op0), m_SCEV(Op1)))) {
3982 // Break (C * (a + b + c)) into C*a + C*b + C*c.
3983 C = C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0;
3984 const SCEV *Remainder = CollectSubexprs(Op1, C, Ops, L, SE, Depth + 1);
3985 if (Remainder)
3986 Ops.push_back(SE.getMulExpr(C, Remainder));
3987 return nullptr;
3988 }
3989 return S;
3990}
3991
3992/// Return true if the SCEV represents a value that may end up as a
3993/// post-increment operation.
3995 LSRUse &LU, const SCEV *S, const Loop *L,
3996 ScalarEvolution &SE) {
3997 if (LU.Kind != LSRUse::Address ||
3998 !LU.AccessTy.getType()->isIntOrIntVectorTy())
3999 return false;
4000 const SCEV *Start;
4001 if (!match(S, m_scev_AffineAddRec(m_SCEV(Start), m_SCEVConstant())))
4002 return false;
4003 // Check if a post-indexed load/store can be used.
4004 if (TTI.isIndexedLoadLegal(TTI.MIM_PostInc, S->getType()) ||
4005 TTI.isIndexedStoreLegal(TTI.MIM_PostInc, S->getType())) {
4006 if (!isa<SCEVConstant>(Start) && SE.isLoopInvariant(Start, L))
4007 return true;
4008 }
4009 return false;
4010}
4011
4012/// Helper function for LSRInstance::GenerateReassociations.
4013void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
4014 const Formula &Base,
4015 unsigned Depth, size_t Idx,
4016 bool IsScaledReg) {
4017 const SCEV *BaseReg = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
4018 // Don't generate reassociations for the base register of a value that
4019 // may generate a post-increment operator. The reason is that the
4020 // reassociations cause extra base+register formula to be created,
4021 // and possibly chosen, but the post-increment is more efficient.
4022 if (AMK == TTI::AMK_PostIndexed && mayUsePostIncMode(TTI, LU, BaseReg, L, SE))
4023 return;
4025 const SCEV *Remainder = CollectSubexprs(BaseReg, nullptr, AddOps, L, SE);
4026 if (Remainder)
4027 AddOps.push_back(Remainder);
4028
4029 if (AddOps.size() == 1)
4030 return;
4031
4033 JE = AddOps.end();
4034 J != JE; ++J) {
4035 // Loop-variant "unknown" values are uninteresting; we won't be able to
4036 // do anything meaningful with them.
4037 if (isa<SCEVUnknown>(*J) && !SE.isLoopInvariant(*J, L))
4038 continue;
4039
4040 // Don't pull a constant into a register if the constant could be folded
4041 // into an immediate field.
4042 if (isAlwaysFoldable(Opts, TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
4043 LU.AccessTy, *J, Base.getNumRegs() > 1))
4044 continue;
4045
4046 // Collect all operands except *J.
4047 SmallVector<SCEVUse, 8> InnerAddOps(std::as_const(AddOps).begin(), J);
4048 InnerAddOps.append(std::next(J), std::as_const(AddOps).end());
4049
4050 // Don't leave just a constant behind in a register if the constant could
4051 // be folded into an immediate field.
4052 if (InnerAddOps.size() == 1 &&
4053 isAlwaysFoldable(Opts, TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
4054 LU.AccessTy, InnerAddOps[0], Base.getNumRegs() > 1))
4055 continue;
4056
4057 const SCEV *InnerSum = SE.getAddExpr(InnerAddOps);
4058 if (InnerSum->isZero())
4059 continue;
4060 Formula F = Base;
4061
4062 if (F.UnfoldedOffset.isNonZero() && F.UnfoldedOffset.isScalable())
4063 continue;
4064
4065 // Add the remaining pieces of the add back into the new formula.
4066 const SCEVConstant *InnerSumSC = dyn_cast<SCEVConstant>(InnerSum);
4067 if (InnerSumSC && SE.getTypeSizeInBits(InnerSumSC->getType()) <= 64 &&
4068 TTI.isLegalAddImmediate((uint64_t)F.UnfoldedOffset.getFixedValue() +
4069 InnerSumSC->getValue()->getZExtValue())) {
4070 F.UnfoldedOffset =
4071 Immediate::getFixed((uint64_t)F.UnfoldedOffset.getFixedValue() +
4072 InnerSumSC->getValue()->getZExtValue());
4073 if (IsScaledReg) {
4074 F.ScaledReg = nullptr;
4075 F.Scale = 0;
4076 } else
4077 F.BaseRegs.erase(F.BaseRegs.begin() + Idx);
4078 } else if (IsScaledReg)
4079 F.ScaledReg = InnerSum;
4080 else
4081 F.BaseRegs[Idx] = InnerSum;
4082
4083 // Add J as its own register, or an unfolded immediate.
4084 const SCEVConstant *SC = dyn_cast<SCEVConstant>(*J);
4085 if (SC && SE.getTypeSizeInBits(SC->getType()) <= 64 &&
4086 TTI.isLegalAddImmediate((uint64_t)F.UnfoldedOffset.getFixedValue() +
4087 SC->getValue()->getZExtValue()))
4088 F.UnfoldedOffset =
4089 Immediate::getFixed((uint64_t)F.UnfoldedOffset.getFixedValue() +
4090 SC->getValue()->getZExtValue());
4091 else
4092 F.BaseRegs.push_back(*J);
4093 // We may have changed the number of register in base regs, adjust the
4094 // formula accordingly.
4095 F.canonicalize(*L);
4096
4097 if (InsertFormula(LU, LUIdx, F))
4098 // If that formula hadn't been seen before, recurse to find more like
4099 // it.
4100 // Add check on Log16(AddOps.size()) - same as Log2_32(AddOps.size()) >> 2)
4101 // Because just Depth is not enough to bound compile time.
4102 // This means that every time AddOps.size() is greater 16^x we will add
4103 // x to Depth.
4104 GenerateReassociations(LU, LUIdx, LU.Formulae.back(),
4105 Depth + 1 + (Log2_32(AddOps.size()) >> 2));
4106 }
4107}
4108
4109/// Split out subexpressions from adds and the bases of addrecs.
4110void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx,
4111 Formula Base, unsigned Depth) {
4112 assert(Base.isCanonical(*L) && "Input must be in the canonical form");
4113 // Arbitrarily cap recursion to protect compile time.
4114 if (Depth >= 3)
4115 return;
4116
4117 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
4118 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, i);
4119
4120 if (Base.Scale == 1)
4121 GenerateReassociationsImpl(LU, LUIdx, Base, Depth,
4122 /* Idx */ -1, /* IsScaledReg */ true);
4123}
4124
4125/// Generate a formula consisting of all of the loop-dominating registers added
4126/// into a single register.
4127void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
4128 Formula Base) {
4129 // This method is only interesting on a plurality of registers.
4130 if (Base.BaseRegs.size() + (Base.Scale == 1) +
4131 (Base.UnfoldedOffset.isNonZero()) <=
4132 1)
4133 return;
4134
4135 // Flatten the representation, i.e., reg1 + 1*reg2 => reg1 + reg2, before
4136 // processing the formula.
4137 Base.unscale();
4139 Formula NewBase = Base;
4140 NewBase.BaseRegs.clear();
4141 Type *CombinedIntegerType = nullptr;
4142 for (const SCEV *BaseReg : Base.BaseRegs) {
4143 if (SE.properlyDominates(BaseReg, L->getHeader()) &&
4144 !SE.hasComputableLoopEvolution(BaseReg, L)) {
4145 if (!CombinedIntegerType)
4146 CombinedIntegerType = SE.getEffectiveSCEVType(BaseReg->getType());
4147 Ops.push_back(BaseReg);
4148 }
4149 else
4150 NewBase.BaseRegs.push_back(BaseReg);
4151 }
4152
4153 // If no register is relevant, we're done.
4154 if (Ops.size() == 0)
4155 return;
4156
4157 // Utility function for generating the required variants of the combined
4158 // registers.
4159 auto GenerateFormula = [&](const SCEV *Sum) {
4160 Formula F = NewBase;
4161
4162 // TODO: If Sum is zero, it probably means ScalarEvolution missed an
4163 // opportunity to fold something. For now, just ignore such cases
4164 // rather than proceed with zero in a register.
4165 if (Sum->isZero())
4166 return;
4167
4168 F.BaseRegs.push_back(Sum);
4169 F.canonicalize(*L);
4170 (void)InsertFormula(LU, LUIdx, F);
4171 };
4172
4173 // If we collected at least two registers, generate a formula combining them.
4174 if (Ops.size() > 1) {
4175 SmallVector<SCEVUse, 4> OpsCopy(Ops); // Don't let SE modify Ops.
4176 GenerateFormula(SE.getAddExpr(OpsCopy));
4177 }
4178
4179 // If we have an unfolded offset, generate a formula combining it with the
4180 // registers collected.
4181 if (NewBase.UnfoldedOffset.isNonZero() && NewBase.UnfoldedOffset.isFixed()) {
4182 assert(CombinedIntegerType && "Missing a type for the unfolded offset");
4183 Ops.push_back(SE.getConstant(CombinedIntegerType,
4184 NewBase.UnfoldedOffset.getFixedValue(), true));
4185 NewBase.UnfoldedOffset = Immediate::getFixed(0);
4186 GenerateFormula(SE.getAddExpr(Ops));
4187 }
4188}
4189
4190/// Helper function for LSRInstance::GenerateSymbolicOffsets.
4191void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
4192 const Formula &Base, size_t Idx,
4193 bool IsScaledReg) {
4194 SCEVUse G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
4195 GlobalValue *GV = ExtractSymbol(G, SE);
4196 if (G->isZero() || !GV)
4197 return;
4198 Formula F = Base;
4199 F.BaseGV = GV;
4200 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
4201 return;
4202 if (IsScaledReg)
4203 F.ScaledReg = G;
4204 else
4205 F.BaseRegs[Idx] = G;
4206 (void)InsertFormula(LU, LUIdx, F);
4207}
4208
4209/// Generate reuse formulae using symbolic offsets.
4210void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
4211 Formula Base) {
4212 // We can't add a symbolic offset if the address already contains one.
4213 if (Base.BaseGV) return;
4214
4215 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
4216 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, i);
4217 if (Base.Scale == 1)
4218 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1,
4219 /* IsScaledReg */ true);
4220}
4221
4222/// Helper function for LSRInstance::GenerateConstantOffsets.
4223void LSRInstance::GenerateConstantOffsetsImpl(
4224 LSRUse &LU, unsigned LUIdx, const Formula &Base,
4225 const SmallVectorImpl<Immediate> &Worklist, size_t Idx, bool IsScaledReg) {
4226
4227 auto GenerateOffset = [&](const SCEV *G, Immediate Offset) {
4228 Formula F = Base;
4229 if (!Base.BaseOffset.isCompatibleImmediate(Offset))
4230 return;
4231 F.BaseOffset = Base.BaseOffset.subUnsigned(Offset);
4232
4233 if (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) {
4234 // Add the offset to the base register.
4235 const SCEV *NewOffset = Offset.getSCEV(SE, G->getType());
4236 const SCEV *NewG = SE.getAddExpr(NewOffset, G);
4237 // If it cancelled out, drop the base register, otherwise update it.
4238 if (NewG->isZero()) {
4239 if (IsScaledReg) {
4240 F.Scale = 0;
4241 F.ScaledReg = nullptr;
4242 } else
4243 F.deleteBaseReg(F.BaseRegs[Idx]);
4244 F.canonicalize(*L);
4245 } else if (IsScaledReg)
4246 F.ScaledReg = NewG;
4247 else
4248 F.BaseRegs[Idx] = NewG;
4249
4250 (void)InsertFormula(LU, LUIdx, F);
4251 }
4252 };
4253
4254 SCEVUse G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
4255
4256 // With constant offsets and constant steps, we can generate pre-inc
4257 // accesses by having the offset equal the step. So, for access #0 with a
4258 // step of 8, we generate a G - 8 base which would require the first access
4259 // to be ((G - 8) + 8),+,8. The pre-indexed access then updates the pointer
4260 // for itself and hopefully becomes the base for other accesses. This means
4261 // means that a single pre-indexed access can be generated to become the new
4262 // base pointer for each iteration of the loop, resulting in no extra add/sub
4263 // instructions for pointer updating.
4264 if ((AMK & TTI::AMK_PreIndexed) && LU.Kind == LSRUse::Address) {
4265 const APInt *StepInt;
4266 if (match(G, m_scev_AffineAddRec(m_SCEV(), m_scev_APInt(StepInt)))) {
4267 int64_t Step = StepInt->isNegative() ? StepInt->getSExtValue()
4268 : StepInt->getZExtValue();
4269
4270 for (Immediate Offset : Worklist) {
4271 if (Offset.isFixed()) {
4272 Offset = Immediate::getFixed(Offset.getFixedValue() - Step);
4273 GenerateOffset(G, Offset);
4274 }
4275 }
4276 }
4277 }
4278 for (Immediate Offset : Worklist)
4279 GenerateOffset(G, Offset);
4280
4281 // TODO: It likely makes sense to extract the immediate corresponding to the
4282 // access type (i.e., set PreferScalable to AccessTy.MemTy &&
4283 // AccessTy.MemTy->isScalableTy()).
4284 Immediate Imm = extractImmediate(Opts, G, SE, /*PreferScalable=*/false);
4285 if (G->isZero() || Imm.isZero() ||
4286 !Base.BaseOffset.isCompatibleImmediate(Imm))
4287 return;
4288 Formula F = Base;
4289 F.BaseOffset = F.BaseOffset.addUnsigned(Imm);
4290 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
4291 return;
4292 if (IsScaledReg) {
4293 F.ScaledReg = G;
4294 } else {
4295 F.BaseRegs[Idx] = G;
4296 // We may generate non canonical Formula if G is a recurrent expr reg
4297 // related with current loop while F.ScaledReg is not.
4298 F.canonicalize(*L);
4299 }
4300 (void)InsertFormula(LU, LUIdx, F);
4301}
4302
4303/// GenerateConstantOffsets - Generate reuse formulae using symbolic offsets.
4304void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx,
4305 Formula Base) {
4306 // TODO: For now, just add the min and max offset, because it usually isn't
4307 // worthwhile looking at everything inbetween.
4309 Worklist.push_back(LU.MinOffset);
4310 if (LU.MaxOffset != LU.MinOffset)
4311 Worklist.push_back(LU.MaxOffset);
4312
4313 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
4314 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, i);
4315 if (Base.Scale == 1)
4316 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1,
4317 /* IsScaledReg */ true);
4318}
4319
4320/// For ICmpZero, check to see if we can scale up the comparison. For example, x
4321/// == y -> x*c == y*c.
4322void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
4323 Formula Base) {
4324 if (LU.Kind != LSRUse::ICmpZero) return;
4325
4326 // Determine the integer type for the base formula.
4327 Type *IntTy = Base.getType();
4328 if (!IntTy) return;
4329 if (SE.getTypeSizeInBits(IntTy) > 64) return;
4330
4331 // Don't do this if there is more than one offset.
4332 if (LU.MinOffset != LU.MaxOffset) return;
4333
4334 // Check if transformation is valid. It is illegal to multiply pointer.
4335 if (Base.ScaledReg && Base.ScaledReg->getType()->isPointerTy())
4336 return;
4337 for (const SCEV *BaseReg : Base.BaseRegs)
4338 if (BaseReg->getType()->isPointerTy())
4339 return;
4340 assert(!Base.BaseGV && "ICmpZero use is not legal!");
4341
4342 // Check each interesting stride.
4343 for (int64_t Factor : Factors) {
4344 // Check that Factor can be represented by IntTy
4345 if (!ConstantInt::isValueValidForType(IntTy, Factor))
4346 continue;
4347 // Check that the multiplication doesn't overflow.
4348 if (Base.BaseOffset.isMin() && Factor == -1)
4349 continue;
4350 // Not supporting scalable immediates.
4351 if (Base.BaseOffset.isNonZero() && Base.BaseOffset.isScalable())
4352 continue;
4353 Immediate NewBaseOffset = Base.BaseOffset.mulUnsigned(Factor);
4354 assert(Factor != 0 && "Zero factor not expected!");
4355 if (NewBaseOffset.getFixedValue() / Factor !=
4356 Base.BaseOffset.getFixedValue())
4357 continue;
4358 // If the offset will be truncated at this use, check that it is in bounds.
4359 if (!IntTy->isPointerTy() &&
4360 !ConstantInt::isValueValidForType(IntTy, NewBaseOffset.getFixedValue()))
4361 continue;
4362
4363 // Check that multiplying with the use offset doesn't overflow.
4364 Immediate Offset = LU.MinOffset;
4365 if (Offset.isMin() && Factor == -1)
4366 continue;
4367 Offset = Offset.mulUnsigned(Factor);
4368 if (Offset.getFixedValue() / Factor != LU.MinOffset.getFixedValue())
4369 continue;
4370 // If the offset will be truncated at this use, check that it is in bounds.
4371 if (!IntTy->isPointerTy() &&
4372 !ConstantInt::isValueValidForType(IntTy, Offset.getFixedValue()))
4373 continue;
4374
4375 Formula F = Base;
4376 F.BaseOffset = NewBaseOffset;
4377
4378 // Check that this scale is legal.
4379 if (!isLegalUse(TTI, Offset, Offset, LU.Kind, LU.AccessTy, F))
4380 continue;
4381
4382 // Compensate for the use having MinOffset built into it.
4383 F.BaseOffset = F.BaseOffset.addUnsigned(Offset).subUnsigned(LU.MinOffset);
4384
4385 const SCEV *FactorS = SE.getConstant(IntTy, Factor);
4386
4387 // Check that multiplying with each base register doesn't overflow.
4388 for (size_t i = 0, e = F.BaseRegs.size(); i != e; ++i) {
4389 F.BaseRegs[i] = SE.getMulExpr(F.BaseRegs[i], FactorS);
4390 if (getExactSDiv(F.BaseRegs[i], FactorS, SE) != Base.BaseRegs[i])
4391 goto next;
4392 }
4393
4394 // Check that multiplying with the scaled register doesn't overflow.
4395 if (F.ScaledReg) {
4396 F.ScaledReg = SE.getMulExpr(F.ScaledReg, FactorS);
4397 if (getExactSDiv(F.ScaledReg, FactorS, SE) != Base.ScaledReg)
4398 continue;
4399 }
4400
4401 // Check that multiplying with the unfolded offset doesn't overflow.
4402 if (F.UnfoldedOffset.isNonZero()) {
4403 if (F.UnfoldedOffset.isMin() && Factor == -1)
4404 continue;
4405 F.UnfoldedOffset = F.UnfoldedOffset.mulUnsigned(Factor);
4406 if (F.UnfoldedOffset.getFixedValue() / Factor !=
4407 Base.UnfoldedOffset.getFixedValue())
4408 continue;
4409 // If the offset will be truncated, check that it is in bounds.
4411 IntTy, F.UnfoldedOffset.getFixedValue()))
4412 continue;
4413 }
4414
4415 // If we make it here and it's legal, add it.
4416 (void)InsertFormula(LU, LUIdx, F);
4417 next:;
4418 }
4419}
4420
4421/// Generate stride factor reuse formulae by making use of scaled-offset address
4422/// modes, for example.
4423void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
4424 // Determine the integer type for the base formula.
4425 Type *IntTy = Base.getType();
4426 if (!IntTy) return;
4427
4428 // If this Formula already has a scaled register, we can't add another one.
4429 // Try to unscale the formula to generate a better scale.
4430 if (Base.Scale != 0 && !Base.unscale())
4431 return;
4432
4433 assert(Base.Scale == 0 && "unscale did not did its job!");
4434
4435 // Check each interesting stride.
4436 for (int64_t Factor : Factors) {
4437 Base.Scale = Factor;
4438 Base.HasBaseReg = Base.BaseRegs.size() > 1;
4439 // Check whether this scale is going to be legal.
4440 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
4441 Base)) {
4442 // As a special-case, handle special out-of-loop Basic users specially.
4443 // TODO: Reconsider this special case.
4444 if (LU.Kind == LSRUse::Basic &&
4445 isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LSRUse::Special,
4446 LU.AccessTy, Base) &&
4447 LU.AllFixupsOutsideLoop)
4448 LU.Kind = LSRUse::Special;
4449 else
4450 continue;
4451 }
4452 // For an ICmpZero, negating a solitary base register won't lead to
4453 // new solutions.
4454 if (LU.Kind == LSRUse::ICmpZero && !Base.HasBaseReg &&
4455 Base.BaseOffset.isZero() && !Base.BaseGV)
4456 continue;
4457 // For each addrec base reg, if its loop is current loop, apply the scale.
4458 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) {
4459 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Base.BaseRegs[i]);
4460 if (AR && (AR->getLoop() == L || LU.AllFixupsOutsideLoop)) {
4461 const SCEV *FactorS = SE.getConstant(IntTy, Factor);
4462 if (FactorS->isZero())
4463 continue;
4464 // Divide out the factor, ignoring high bits, since we'll be
4465 // scaling the value back up in the end.
4466 if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true))
4467 if (!Quotient->isZero()) {
4468 // TODO: This could be optimized to avoid all the copying.
4469 Formula F = Base;
4470 F.ScaledReg = Quotient;
4471 F.deleteBaseReg(F.BaseRegs[i]);
4472 // The canonical representation of 1*reg is reg, which is already in
4473 // Base. In that case, do not try to insert the formula, it will be
4474 // rejected anyway.
4475 if (F.Scale == 1 && (F.BaseRegs.empty() ||
4476 (AR->getLoop() != L && LU.AllFixupsOutsideLoop)))
4477 continue;
4478 // If AllFixupsOutsideLoop is true and F.Scale is 1, we may generate
4479 // non canonical Formula with ScaledReg's loop not being L.
4480 if (F.Scale == 1 && LU.AllFixupsOutsideLoop)
4481 F.canonicalize(*L);
4482 (void)InsertFormula(LU, LUIdx, F);
4483 }
4484 }
4485 }
4486 }
4487}
4488
4489/// Extend/Truncate \p Expr to \p ToTy considering post-inc uses in \p Loops.
4490/// For all PostIncLoopSets in \p Loops, first de-normalize \p Expr, then
4491/// perform the extension/truncate and normalize again, as the normalized form
4492/// can result in folds that are not valid in the post-inc use contexts. The
4493/// expressions for all PostIncLoopSets must match, otherwise return nullptr.
4494static const SCEV *
4496 const SCEV *Expr, Type *ToTy,
4497 ScalarEvolution &SE) {
4498 const SCEV *Result = nullptr;
4499 for (auto &L : Loops) {
4500 auto *DenormExpr = denormalizeForPostIncUse(Expr, L, SE);
4501 const SCEV *NewDenormExpr = SE.getAnyExtendExpr(DenormExpr, ToTy);
4502 const SCEV *New = normalizeForPostIncUse(NewDenormExpr, L, SE);
4503 if (!New || (Result && New != Result))
4504 return nullptr;
4505 Result = New;
4506 }
4507
4508 assert(Result && "failed to create expression");
4509 return Result;
4510}
4511
4512/// Generate reuse formulae from different IV types.
4513void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
4514 // Don't bother truncating symbolic values.
4515 if (Base.BaseGV) return;
4516
4517 // Determine the integer type for the base formula.
4518 Type *DstTy = Base.getType();
4519 if (!DstTy) return;
4520 if (DstTy->isPointerTy())
4521 return;
4522
4523 // It is invalid to extend a pointer type so exit early if ScaledReg or
4524 // any of the BaseRegs are pointers.
4525 if (Base.ScaledReg && Base.ScaledReg->getType()->isPointerTy())
4526 return;
4527 if (any_of(Base.BaseRegs,
4528 [](const SCEV *S) { return S->getType()->isPointerTy(); }))
4529 return;
4530
4532 for (auto &LF : LU.Fixups)
4533 Loops.push_back(LF.PostIncLoops);
4534
4535 for (Type *SrcTy : Types) {
4536 if (SrcTy != DstTy && TTI.isTruncateFree(SrcTy, DstTy)) {
4537 Formula F = Base;
4538
4539 // Sometimes SCEV is able to prove zero during ext transform. It may
4540 // happen if SCEV did not do all possible transforms while creating the
4541 // initial node (maybe due to depth limitations), but it can do them while
4542 // taking ext.
4543 if (F.ScaledReg) {
4544 const SCEV *NewScaledReg =
4545 getAnyExtendConsideringPostIncUses(Loops, F.ScaledReg, SrcTy, SE);
4546 if (!NewScaledReg || NewScaledReg->isZero())
4547 continue;
4548 F.ScaledReg = NewScaledReg;
4549 }
4550 bool HasZeroBaseReg = false;
4551 for (const SCEV *&BaseReg : F.BaseRegs) {
4552 const SCEV *NewBaseReg =
4553 getAnyExtendConsideringPostIncUses(Loops, BaseReg, SrcTy, SE);
4554 if (!NewBaseReg || NewBaseReg->isZero()) {
4555 HasZeroBaseReg = true;
4556 break;
4557 }
4558 BaseReg = NewBaseReg;
4559 }
4560 if (HasZeroBaseReg)
4561 continue;
4562
4563 // TODO: This assumes we've done basic processing on all uses and
4564 // have an idea what the register usage is.
4565 if (!F.hasRegsUsedByUsesOtherThan(LUIdx, RegUses))
4566 continue;
4567
4568 F.canonicalize(*L);
4569 (void)InsertFormula(LU, LUIdx, F);
4570 }
4571 }
4572}
4573
4574namespace {
4575
4576/// Helper class for GenerateCrossUseConstantOffsets. It's used to defer
4577/// modifications so that the search phase doesn't have to worry about the data
4578/// structures moving underneath it.
4579struct WorkItem {
4580 size_t LUIdx;
4581 Immediate Imm;
4582 const SCEV *OrigReg;
4583
4584 WorkItem(size_t LI, Immediate I, const SCEV *R)
4585 : LUIdx(LI), Imm(I), OrigReg(R) {}
4586
4587 void print(raw_ostream &OS) const;
4588 void dump() const;
4589};
4590
4591} // end anonymous namespace
4592
4593#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4594void WorkItem::print(raw_ostream &OS) const {
4595 OS << "in formulae referencing " << *OrigReg << " in use " << LUIdx
4596 << " , add offset " << Imm;
4597}
4598
4599LLVM_DUMP_METHOD void WorkItem::dump() const {
4600 print(errs()); errs() << '\n';
4601}
4602#endif
4603
4604/// Look for registers which are a constant distance apart and try to form reuse
4605/// opportunities between them.
4606void LSRInstance::GenerateCrossUseConstantOffsets() {
4607 // Group the registers by their value without any added constant offset.
4608 using ImmMapTy = std::map<Immediate, const SCEV *, KeyOrderTargetImmediate>;
4609
4610 DenseMap<const SCEV *, ImmMapTy> Map;
4611 DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap;
4613 for (const SCEV *Use : RegUses) {
4614 SCEVUse Reg = Use; // Make a copy for extractImmediate to modify.
4615 // TODO: Extract both scalable and fixed immediates (if present)?
4616 Immediate Imm = extractImmediate(Opts, Reg, SE);
4617 auto Pair = Map.try_emplace(Reg);
4618 if (Pair.second)
4619 Sequence.push_back(Reg);
4620 Pair.first->second.insert(std::make_pair(Imm, Use));
4621 UsedByIndicesMap[Reg] |= RegUses.getUsedByIndices(Use);
4622 }
4623
4624 // Now examine each set of registers with the same base value. Build up
4625 // a list of work to do and do the work in a separate step so that we're
4626 // not adding formulae and register counts while we're searching.
4627 SmallVector<WorkItem, 32> WorkItems;
4628 SmallSet<std::pair<size_t, Immediate>, 32, KeyOrderSizeTAndImmediate>
4629 UniqueItems;
4630 for (const SCEV *Reg : Sequence) {
4631 const ImmMapTy &Imms = Map.find(Reg)->second;
4632
4633 // It's not worthwhile looking for reuse if there's only one offset.
4634 if (Imms.size() == 1)
4635 continue;
4636
4637 LLVM_DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << ':';
4638 for (const auto &Entry
4639 : Imms) dbgs()
4640 << ' ' << Entry.first;
4641 dbgs() << '\n');
4642
4643 // Examine each offset.
4644 for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end();
4645 J != JE; ++J) {
4646 const SCEV *OrigReg = J->second;
4647
4648 Immediate JImm = J->first;
4649 const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(OrigReg);
4650
4651 if (!isa<SCEVConstant>(OrigReg) &&
4652 UsedByIndicesMap[Reg].count() == 1) {
4653 LLVM_DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg
4654 << '\n');
4655 continue;
4656 }
4657
4658 // Conservatively examine offsets between this orig reg a few selected
4659 // other orig regs.
4660 Immediate First = Imms.begin()->first;
4661 Immediate Last = std::prev(Imms.end())->first;
4662 if (!First.isCompatibleImmediate(Last)) {
4663 LLVM_DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg
4664 << "\n");
4665 continue;
4666 }
4667 // Only scalable if both terms are scalable, or if one is scalable and
4668 // the other is 0.
4669 bool Scalable = First.isScalable() || Last.isScalable();
4670 int64_t FI = First.getKnownMinValue();
4671 int64_t LI = Last.getKnownMinValue();
4672 // Compute (First + Last) / 2 without overflow using the fact that
4673 // First + Last = 2 * (First + Last) + (First ^ Last).
4674 int64_t Avg = (FI & LI) + ((FI ^ LI) >> 1);
4675 // If the result is negative and FI is odd and LI even (or vice versa),
4676 // we rounded towards -inf. Add 1 in that case, to round towards 0.
4677 Avg = Avg + ((FI ^ LI) & ((uint64_t)Avg >> 63));
4678 ImmMapTy::const_iterator OtherImms[] = {
4679 Imms.begin(), std::prev(Imms.end()),
4680 Imms.lower_bound(Immediate::get(Avg, Scalable))};
4681 for (const auto &M : OtherImms) {
4682 if (M == J || M == JE) continue;
4683 if (!JImm.isCompatibleImmediate(M->first))
4684 continue;
4685
4686 // Compute the difference between the two.
4687 Immediate Imm = JImm.subUnsigned(M->first);
4688 for (unsigned LUIdx : UsedByIndices.set_bits())
4689 // Make a memo of this use, offset, and register tuple.
4690 if (UniqueItems.insert(std::make_pair(LUIdx, Imm)).second)
4691 WorkItems.push_back(WorkItem(LUIdx, Imm, OrigReg));
4692 }
4693 }
4694 }
4695
4696 Map.clear();
4697 Sequence.clear();
4698 UsedByIndicesMap.clear();
4699 UniqueItems.clear();
4700
4701 // Now iterate through the worklist and add new formulae.
4702 for (const WorkItem &WI : WorkItems) {
4703 size_t LUIdx = WI.LUIdx;
4704 LSRUse &LU = Uses[LUIdx];
4705 Immediate Imm = WI.Imm;
4706 const SCEV *OrigReg = WI.OrigReg;
4707
4708 Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType());
4709 const SCEV *NegImmS = Imm.getNegativeSCEV(SE, IntTy);
4710 unsigned BitWidth = SE.getTypeSizeInBits(IntTy);
4711
4712 // TODO: Use a more targeted data structure.
4713 for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) {
4714 Formula F = LU.Formulae[L];
4715 // FIXME: The code for the scaled and unscaled registers looks
4716 // very similar but slightly different. Investigate if they
4717 // could be merged. That way, we would not have to unscale the
4718 // Formula.
4719 F.unscale();
4720 // Use the immediate in the scaled register.
4721 if (F.ScaledReg == OrigReg) {
4722 if (!F.BaseOffset.isCompatibleImmediate(Imm))
4723 continue;
4724 Immediate Offset = F.BaseOffset.addUnsigned(Imm.mulUnsigned(F.Scale));
4725 // Don't create 50 + reg(-50).
4726 const SCEV *S = Offset.getNegativeSCEV(SE, IntTy);
4727 if (F.referencesReg(S))
4728 continue;
4729 Formula NewF = F;
4730 NewF.BaseOffset = Offset;
4731 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
4732 NewF))
4733 continue;
4734 NewF.ScaledReg = SE.getAddExpr(NegImmS, NewF.ScaledReg);
4735
4736 // If the new scale is a constant in a register, and adding the constant
4737 // value to the immediate would produce a value closer to zero than the
4738 // immediate itself, then the formula isn't worthwhile.
4739 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewF.ScaledReg)) {
4740 // FIXME: Do we need to do something for scalable immediates here?
4741 // A scalable SCEV won't be constant, but we might still have
4742 // something in the offset? Bail out for now to be safe.
4743 if (NewF.BaseOffset.isNonZero() && NewF.BaseOffset.isScalable())
4744 continue;
4745 if (C->getValue()->isNegative() !=
4746 (NewF.BaseOffset.isLessThanZero()) &&
4747 (C->getAPInt().abs() * APInt(BitWidth, F.Scale))
4748 .ule(std::abs(NewF.BaseOffset.getFixedValue())))
4749 continue;
4750 }
4751
4752 // OK, looks good.
4753 NewF.canonicalize(*this->L);
4754 (void)InsertFormula(LU, LUIdx, NewF);
4755 } else {
4756 // Use the immediate in a base register.
4757 for (size_t N = 0, NE = F.BaseRegs.size(); N != NE; ++N) {
4758 const SCEV *BaseReg = F.BaseRegs[N];
4759 if (BaseReg != OrigReg)
4760 continue;
4761 Formula NewF = F;
4762 if (!NewF.BaseOffset.isCompatibleImmediate(Imm) ||
4763 !NewF.UnfoldedOffset.isCompatibleImmediate(Imm) ||
4764 !NewF.BaseOffset.isCompatibleImmediate(NewF.UnfoldedOffset))
4765 continue;
4766 NewF.BaseOffset = NewF.BaseOffset.addUnsigned(Imm);
4767 if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset,
4768 LU.Kind, LU.AccessTy, NewF)) {
4769 if (AMK == TTI::AMK_PostIndexed &&
4770 mayUsePostIncMode(TTI, LU, OrigReg, this->L, SE))
4771 continue;
4772 Immediate NewUnfoldedOffset = NewF.UnfoldedOffset.addUnsigned(Imm);
4773 if (!isLegalAddImmediate(TTI, NewUnfoldedOffset))
4774 continue;
4775 NewF = F;
4776 NewF.UnfoldedOffset = NewUnfoldedOffset;
4777 }
4778 NewF.BaseRegs[N] = SE.getAddExpr(NegImmS, BaseReg);
4779
4780 // If the new formula has a constant in a register, and adding the
4781 // constant value to the immediate would produce a value closer to
4782 // zero than the immediate itself, then the formula isn't worthwhile.
4783 for (const SCEV *NewReg : NewF.BaseRegs)
4784 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewReg)) {
4785 if (NewF.BaseOffset.isNonZero() && NewF.BaseOffset.isScalable())
4786 goto skip_formula;
4787 if ((C->getAPInt() + NewF.BaseOffset.getFixedValue())
4788 .abs()
4789 .slt(std::abs(NewF.BaseOffset.getFixedValue())) &&
4790 (C->getAPInt() + NewF.BaseOffset.getFixedValue())
4791 .countr_zero() >=
4793 NewF.BaseOffset.getFixedValue()))
4794 goto skip_formula;
4795 }
4796
4797 // Ok, looks good.
4798 NewF.canonicalize(*this->L);
4799 (void)InsertFormula(LU, LUIdx, NewF);
4800 break;
4801 skip_formula:;
4802 }
4803 }
4804 }
4805 }
4806}
4807
4808/// Generate formulae for each use.
4809void
4810LSRInstance::GenerateAllReuseFormulae() {
4811 // This is split into multiple loops so that hasRegsUsedByUsesOtherThan
4812 // queries are more precise.
4813 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4814 LSRUse &LU = Uses[LUIdx];
4815 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4816 GenerateReassociations(LU, LUIdx, LU.Formulae[i]);
4817 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4818 GenerateCombinations(LU, LUIdx, LU.Formulae[i]);
4819 }
4820 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4821 LSRUse &LU = Uses[LUIdx];
4822 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4823 GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]);
4824 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4825 GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]);
4826 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4827 GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]);
4828 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4829 GenerateScales(LU, LUIdx, LU.Formulae[i]);
4830 }
4831 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4832 LSRUse &LU = Uses[LUIdx];
4833 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4834 GenerateTruncates(LU, LUIdx, LU.Formulae[i]);
4835 }
4836
4837 GenerateCrossUseConstantOffsets();
4838
4839 LLVM_DEBUG(dbgs() << "\n"
4840 "After generating reuse formulae:\n";
4841 print_uses(dbgs()));
4842}
4843
4844/// If there are multiple formulae with the same set of registers used
4845/// by other uses, pick the best one and delete the others.
4846void LSRInstance::FilterOutUndesirableDedicatedRegisters() {
4847 DenseSet<const SCEV *> VisitedRegs;
4848 SmallPtrSet<const SCEV *, 16> Regs;
4849 SmallPtrSet<const SCEV *, 16> LoserRegs;
4850#ifndef NDEBUG
4851 bool ChangedFormulae = false;
4852#endif
4853
4854 // Collect the best formula for each unique set of shared registers. This
4855 // is reset for each use.
4856 using BestFormulaeTy = DenseMap<SmallVector<const SCEV *, 4>, size_t>;
4857
4858 BestFormulaeTy BestFormulae;
4859
4860 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4861 LSRUse &LU = Uses[LUIdx];
4862 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
4863 dbgs() << '\n');
4864
4865 bool Any = false;
4866 for (size_t FIdx = 0, NumForms = LU.Formulae.size();
4867 FIdx != NumForms; ++FIdx) {
4868 Formula &F = LU.Formulae[FIdx];
4869
4870 // Some formulas are instant losers. For example, they may depend on
4871 // nonexistent AddRecs from other loops. These need to be filtered
4872 // immediately, otherwise heuristics could choose them over others leading
4873 // to an unsatisfactory solution. Passing LoserRegs into RateFormula here
4874 // avoids the need to recompute this information across formulae using the
4875 // same bad AddRec. Passing LoserRegs is also essential unless we remove
4876 // the corresponding bad register from the Regs set.
4877 Cost CostF(Opts, L, SE, TTI, AMK);
4878 Regs.clear();
4879 CostF.RateFormula(F, Regs, VisitedRegs, LU, HardwareLoopProfitable,
4880 &LoserRegs);
4881 if (CostF.isLoser()) {
4882 // During initial formula generation, undesirable formulae are generated
4883 // by uses within other loops that have some non-trivial address mode or
4884 // use the postinc form of the IV. LSR needs to provide these formulae
4885 // as the basis of rediscovering the desired formula that uses an AddRec
4886 // corresponding to the existing phi. Once all formulae have been
4887 // generated, these initial losers may be pruned.
4888 LLVM_DEBUG(dbgs() << " Filtering loser "; F.print(dbgs());
4889 dbgs() << "\n");
4890 }
4891 else {
4893 for (const SCEV *Reg : F.BaseRegs) {
4894 if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx))
4895 Key.push_back(Reg);
4896 }
4897 if (F.ScaledReg &&
4898 RegUses.isRegUsedByUsesOtherThan(F.ScaledReg, LUIdx))
4899 Key.push_back(F.ScaledReg);
4900 // Unstable sort by host order ok, because this is only used for
4901 // uniquifying.
4902 llvm::sort(Key);
4903
4904 std::pair<BestFormulaeTy::const_iterator, bool> P =
4905 BestFormulae.insert(std::make_pair(Key, FIdx));
4906 if (P.second)
4907 continue;
4908
4909 Formula &Best = LU.Formulae[P.first->second];
4910
4911 Cost CostBest(Opts, L, SE, TTI, AMK);
4912 Regs.clear();
4913 CostBest.RateFormula(Best, Regs, VisitedRegs, LU,
4914 HardwareLoopProfitable);
4915 if (CostF.isLess(CostBest))
4916 std::swap(F, Best);
4917 LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
4918 dbgs() << "\n"
4919 " in favor of formula ";
4920 Best.print(dbgs()); dbgs() << '\n');
4921 }
4922#ifndef NDEBUG
4923 ChangedFormulae = true;
4924#endif
4925 LU.DeleteFormula(F);
4926 --FIdx;
4927 --NumForms;
4928 Any = true;
4929 }
4930
4931 // Now that we've filtered out some formulae, recompute the Regs set.
4932 if (Any)
4933 LU.RecomputeRegs(LUIdx, RegUses);
4934
4935 // Reset this to prepare for the next use.
4936 BestFormulae.clear();
4937 }
4938
4939 LLVM_DEBUG(if (ChangedFormulae) {
4940 dbgs() << "\n"
4941 "After filtering out undesirable candidates:\n";
4942 print_uses(dbgs());
4943 });
4944}
4945
4946/// Estimate the worst-case number of solutions the solver might have to
4947/// consider. It almost never considers this many solutions because it prune the
4948/// search space, but the pruning isn't always sufficient.
4949size_t LSRInstance::EstimateSearchSpaceComplexity() const {
4950 size_t Power = 1;
4951 for (const LSRUse &LU : Uses) {
4952 size_t FSize = LU.Formulae.size();
4953 if (FSize >= Opts.lsr_complexity_limit) {
4954 Power = Opts.lsr_complexity_limit;
4955 break;
4956 }
4957 Power *= FSize;
4958 if (Power >= Opts.lsr_complexity_limit)
4959 break;
4960 }
4961 return Power;
4962}
4963
4964/// When one formula uses a superset of the registers of another formula, it
4965/// won't help reduce register pressure (though it may not necessarily hurt
4966/// register pressure); remove it to simplify the system.
4967void LSRInstance::NarrowSearchSpaceByDetectingSupersets() {
4968 if (EstimateSearchSpaceComplexity() >= Opts.lsr_complexity_limit) {
4969 LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
4970
4971 LLVM_DEBUG(dbgs() << "Narrowing the search space by eliminating formulae "
4972 "which use a superset of registers used by other "
4973 "formulae.\n");
4974
4975 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4976 LSRUse &LU = Uses[LUIdx];
4977 bool Any = false;
4978 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4979 Formula &F = LU.Formulae[i];
4980 if (F.BaseOffset.isNonZero() && F.BaseOffset.isScalable())
4981 continue;
4982 // Look for a formula with a constant or GV in a register. If the use
4983 // also has a formula with that same value in an immediate field,
4984 // delete the one that uses a register.
4986 I = F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) {
4987 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*I)) {
4988 Formula NewF = F;
4989 //FIXME: Formulas should store bitwidth to do wrapping properly.
4990 // See PR41034.
4991 NewF.BaseOffset =
4992 Immediate::getFixed(NewF.BaseOffset.getFixedValue() +
4993 (uint64_t)C->getValue()->getSExtValue());
4994 NewF.BaseRegs.erase(NewF.BaseRegs.begin() +
4995 (I - F.BaseRegs.begin()));
4996 if (LU.HasFormulaWithSameRegs(NewF)) {
4997 LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs());
4998 dbgs() << '\n');
4999 LU.DeleteFormula(F);
5000 --i;
5001 --e;
5002 Any = true;
5003 break;
5004 }
5005 } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(*I)) {
5006 if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue()))
5007 if (!F.BaseGV) {
5008 Formula NewF = F;
5009 NewF.BaseGV = GV;
5010 NewF.BaseRegs.erase(NewF.BaseRegs.begin() +
5011 (I - F.BaseRegs.begin()));
5012 if (LU.HasFormulaWithSameRegs(NewF)) {
5013 LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs());
5014 dbgs() << '\n');
5015 LU.DeleteFormula(F);
5016 --i;
5017 --e;
5018 Any = true;
5019 break;
5020 }
5021 }
5022 }
5023 }
5024 }
5025 if (Any)
5026 LU.RecomputeRegs(LUIdx, RegUses);
5027 }
5028
5029 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5030 }
5031}
5032
5033/// When there are many registers for expressions like A, A+1, A+2, etc.,
5034/// allocate a single register for them.
5035void LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() {
5036 if (EstimateSearchSpaceComplexity() < Opts.lsr_complexity_limit)
5037 return;
5038
5039 LLVM_DEBUG(
5040 dbgs() << "The search space is too complex.\n"
5041 "Narrowing the search space by assuming that uses separated "
5042 "by a constant offset will use the same registers.\n");
5043
5044 // This is especially useful for unrolled loops.
5045
5046 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5047 LSRUse &LU = Uses[LUIdx];
5048 for (const Formula &F : LU.Formulae) {
5049 if (F.BaseOffset.isZero() || (F.Scale != 0 && F.Scale != 1))
5050 continue;
5051 assert((LU.Kind == LSRUse::Address || LU.Kind == LSRUse::ICmpZero) &&
5052 "Only address and cmp uses expected to have nonzero BaseOffset");
5053
5054 LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU);
5055 if (!LUThatHas)
5056 continue;
5057
5058 if (!reconcileNewOffset(*LUThatHas, F.BaseOffset, /*HasBaseReg=*/ false,
5059 LU.Kind, LU.AccessTy))
5060 continue;
5061
5062 LLVM_DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << '\n');
5063
5064 LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
5065 LUThatHas->AllFixupsUnconditional &= LU.AllFixupsUnconditional;
5066
5067 // Transfer the fixups of LU to LUThatHas.
5068 for (LSRFixup &Fixup : LU.Fixups) {
5069 Fixup.Offset += F.BaseOffset;
5070 LUThatHas->pushFixup(Fixup);
5071 LLVM_DEBUG(dbgs() << "New fixup has offset " << Fixup.Offset << '\n');
5072 }
5073
5074#ifndef NDEBUG
5075 Type *FixupType = LUThatHas->Fixups[0].OperandValToReplace->getType();
5076 for (LSRFixup &Fixup : LUThatHas->Fixups)
5077 assert(Fixup.OperandValToReplace->getType() == FixupType &&
5078 "Expected all fixups to have the same type");
5079#endif
5080
5081 // Delete formulae from the new use which are no longer legal.
5082 bool Any = false;
5083 for (size_t i = 0, e = LUThatHas->Formulae.size(); i != e; ++i) {
5084 Formula &F = LUThatHas->Formulae[i];
5085 if (!isLegalUse(TTI, LUThatHas->MinOffset, LUThatHas->MaxOffset,
5086 LUThatHas->Kind, LUThatHas->AccessTy, F)) {
5087 LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n');
5088 LUThatHas->DeleteFormula(F);
5089 --i;
5090 --e;
5091 Any = true;
5092 }
5093 }
5094
5095 if (Any)
5096 LUThatHas->RecomputeRegs(LUThatHas - &Uses.front(), RegUses);
5097
5098 // Delete the old use.
5099 DeleteUse(LU, LUIdx);
5100 --LUIdx;
5101 --NumUses;
5102 break;
5103 }
5104 }
5105
5106 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5107}
5108
5109/// Call FilterOutUndesirableDedicatedRegisters again, if necessary, now that
5110/// we've done more filtering, as it may be able to find more formulae to
5111/// eliminate.
5112void LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){
5113 if (EstimateSearchSpaceComplexity() >= Opts.lsr_complexity_limit) {
5114 LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5115
5116 LLVM_DEBUG(dbgs() << "Narrowing the search space by re-filtering out "
5117 "undesirable dedicated registers.\n");
5118
5119 FilterOutUndesirableDedicatedRegisters();
5120
5121 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5122 }
5123}
5124
5125/// If a LSRUse has multiple formulae with the same ScaledReg and Scale.
5126/// Pick the best one and delete the others.
5127/// This narrowing heuristic is to keep as many formulae with different
5128/// Scale and ScaledReg pair as possible while narrowing the search space.
5129/// The benefit is that it is more likely to find out a better solution
5130/// from a formulae set with more Scale and ScaledReg variations than
5131/// a formulae set with the same Scale and ScaledReg. The picking winner
5132/// reg heuristic will often keep the formulae with the same Scale and
5133/// ScaledReg and filter others, and we want to avoid that if possible.
5134void LSRInstance::NarrowSearchSpaceByFilterFormulaWithSameScaledReg() {
5135 if (EstimateSearchSpaceComplexity() < Opts.lsr_complexity_limit)
5136 return;
5137
5138 LLVM_DEBUG(
5139 dbgs() << "The search space is too complex.\n"
5140 "Narrowing the search space by choosing the best Formula "
5141 "from the Formulae with the same Scale and ScaledReg.\n");
5142
5143 // Map the "Scale * ScaledReg" pair to the best formula of current LSRUse.
5144 using BestFormulaeTy = DenseMap<std::pair<const SCEV *, int64_t>, size_t>;
5145
5146 BestFormulaeTy BestFormulae;
5147#ifndef NDEBUG
5148 bool ChangedFormulae = false;
5149#endif
5150 DenseSet<const SCEV *> VisitedRegs;
5151 SmallPtrSet<const SCEV *, 16> Regs;
5152
5153 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5154 LSRUse &LU = Uses[LUIdx];
5155 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
5156 dbgs() << '\n');
5157
5158 // Return true if Formula FA is better than Formula FB.
5159 auto IsBetterThan = [&](Formula &FA, Formula &FB) {
5160 // First we will try to choose the Formula with fewer new registers.
5161 // For a register used by current Formula, the more the register is
5162 // shared among LSRUses, the less we increase the register number
5163 // counter of the formula.
5164 size_t FARegNum = 0;
5165 for (const SCEV *Reg : FA.BaseRegs) {
5166 const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg);
5167 FARegNum += (NumUses - UsedByIndices.count() + 1);
5168 }
5169 size_t FBRegNum = 0;
5170 for (const SCEV *Reg : FB.BaseRegs) {
5171 const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg);
5172 FBRegNum += (NumUses - UsedByIndices.count() + 1);
5173 }
5174 if (FARegNum != FBRegNum)
5175 return FARegNum < FBRegNum;
5176
5177 // If the new register numbers are the same, choose the Formula with
5178 // less Cost.
5179 Cost CostFA(Opts, L, SE, TTI, AMK);
5180 Cost CostFB(Opts, L, SE, TTI, AMK);
5181 Regs.clear();
5182 CostFA.RateFormula(FA, Regs, VisitedRegs, LU, HardwareLoopProfitable);
5183 Regs.clear();
5184 CostFB.RateFormula(FB, Regs, VisitedRegs, LU, HardwareLoopProfitable);
5185 return CostFA.isLess(CostFB);
5186 };
5187
5188 bool Any = false;
5189 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
5190 ++FIdx) {
5191 Formula &F = LU.Formulae[FIdx];
5192 if (!F.ScaledReg)
5193 continue;
5194 auto P = BestFormulae.insert({{F.ScaledReg, F.Scale}, FIdx});
5195 if (P.second)
5196 continue;
5197
5198 Formula &Best = LU.Formulae[P.first->second];
5199 if (IsBetterThan(F, Best))
5200 std::swap(F, Best);
5201 LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
5202 dbgs() << "\n"
5203 " in favor of formula ";
5204 Best.print(dbgs()); dbgs() << '\n');
5205#ifndef NDEBUG
5206 ChangedFormulae = true;
5207#endif
5208 LU.DeleteFormula(F);
5209 --FIdx;
5210 --NumForms;
5211 Any = true;
5212 }
5213 if (Any)
5214 LU.RecomputeRegs(LUIdx, RegUses);
5215
5216 // Reset this to prepare for the next use.
5217 BestFormulae.clear();
5218 }
5219
5220 LLVM_DEBUG(if (ChangedFormulae) {
5221 dbgs() << "\n"
5222 "After filtering out undesirable candidates:\n";
5223 print_uses(dbgs());
5224 });
5225}
5226
5227/// If we are over the complexity limit, filter out any post-inc prefering
5228/// variables to only post-inc values.
5229void LSRInstance::NarrowSearchSpaceByFilterPostInc() {
5230 if (AMK != TTI::AMK_PostIndexed)
5231 return;
5232 if (EstimateSearchSpaceComplexity() < Opts.lsr_complexity_limit)
5233 return;
5234
5235 LLVM_DEBUG(dbgs() << "The search space is too complex.\n"
5236 "Narrowing the search space by choosing the lowest "
5237 "register Formula for PostInc Uses.\n");
5238
5239 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5240 LSRUse &LU = Uses[LUIdx];
5241
5242 if (LU.Kind != LSRUse::Address)
5243 continue;
5244 if (!TTI.isIndexedLoadLegal(TTI.MIM_PostInc, LU.AccessTy.getType()) &&
5245 !TTI.isIndexedStoreLegal(TTI.MIM_PostInc, LU.AccessTy.getType()))
5246 continue;
5247
5248 size_t MinRegs = std::numeric_limits<size_t>::max();
5249 for (const Formula &F : LU.Formulae)
5250 MinRegs = std::min(F.getNumRegs(), MinRegs);
5251
5252 bool Any = false;
5253 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
5254 ++FIdx) {
5255 Formula &F = LU.Formulae[FIdx];
5256 if (F.getNumRegs() > MinRegs) {
5257 LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
5258 dbgs() << "\n");
5259 LU.DeleteFormula(F);
5260 --FIdx;
5261 --NumForms;
5262 Any = true;
5263 }
5264 }
5265 if (Any)
5266 LU.RecomputeRegs(LUIdx, RegUses);
5267
5268 if (EstimateSearchSpaceComplexity() < Opts.lsr_complexity_limit)
5269 break;
5270 }
5271
5272 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5273}
5274
5275void LSRInstance::NarrowSearchSpaceByMergingUsesOutsideLoop() {
5276 if (EstimateSearchSpaceComplexity() < Opts.lsr_complexity_limit)
5277 return;
5278
5279 LLVM_DEBUG(
5280 dbgs() << "The search space is too complex.\n"
5281 "Narrowing the search space by merging uses with fixups "
5282 "entirely outside the loop with uses inside the loop.\n");
5283
5284 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5285 LSRUse &LU = Uses[LUIdx];
5286 // Don't merge ICmpZero uses outside the loop, as ICmpZero needs to be
5287 // handled specially when expanding.
5288 if (!LU.AllFixupsOutsideLoop || LU.Formulae.empty() ||
5289 LU.Kind == LSRUse::ICmpZero)
5290 continue;
5291
5292 LLVM_DEBUG(dbgs() << " Trying to eliminate use "; LU.print(dbgs());
5293 dbgs() << '\n');
5294
5295 // Find a compatible LSRUse inside the loop that we could merge LU with
5296 LSRUse *LUToMergeWith = nullptr;
5297 const Formula &ThisF = LU.Formulae[0];
5298 for (LSRUse &OtherLU : Uses) {
5299 // Only merge with uses inside the loop
5300 if (OtherLU.AllFixupsOutsideLoop)
5301 continue;
5302 // Can't merge with ICmpZero uses as they're handled specially when
5303 // expanding
5304 if (OtherLU.Kind == LSRUse::ICmpZero)
5305 continue;
5306 // Can't merge with uses without any formulae
5307 if (OtherLU.Formulae.empty())
5308 continue;
5309 // Can't merge if LU's offsets aren't legal for all of OtherLU's formulae
5310 if (any_of(OtherLU.Formulae, [&](const Formula &F) {
5311 return !isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, OtherLU.Kind,
5312 OtherLU.AccessTy, F);
5313 }))
5314 continue;
5315 // We can merge with uses that have the same initial formula. We allow
5316 // merging of uses with different Kind and AccessTy which means that the
5317 // cost may end up being inaccurate, but it's also what we would have
5318 // gotten if we'd ignored uses outside the loop entirely.
5319 const Formula &OtherF = OtherLU.Formulae[0];
5320 if (ThisF.BaseRegs == OtherF.BaseRegs &&
5321 ThisF.ScaledReg == OtherF.ScaledReg &&
5322 ThisF.BaseGV == OtherF.BaseGV && ThisF.Scale == OtherF.Scale &&
5323 ThisF.UnfoldedOffset == OtherF.UnfoldedOffset &&
5324 ThisF.BaseOffset == OtherF.BaseOffset) {
5325 LUToMergeWith = &OtherLU;
5326 break;
5327 }
5328 }
5329 if (!LUToMergeWith)
5330 continue;
5331
5332 LLVM_DEBUG(dbgs() << " Merging with "; LUToMergeWith->print(dbgs());
5333 dbgs() << '\n');
5334
5335 // Copy fixups
5336 for (LSRFixup &Fixup : LU.Fixups) {
5337 LUToMergeWith->pushFixup(Fixup);
5338 }
5339
5340 // Delete the old use.
5341 DeleteUse(LU, LUIdx);
5342 --LUIdx;
5343 --NumUses;
5344 }
5345
5346 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5347}
5348
5349/// The function delete formulas with high registers number expectation.
5350/// Assuming we don't know the value of each formula (already delete
5351/// all inefficient), generate probability of not selecting for each
5352/// register.
5353/// For example,
5354/// Use1:
5355/// reg(a) + reg({0,+,1})
5356/// reg(a) + reg({-1,+,1}) + 1
5357/// reg({a,+,1})
5358/// Use2:
5359/// reg(b) + reg({0,+,1})
5360/// reg(b) + reg({-1,+,1}) + 1
5361/// reg({b,+,1})
5362/// Use3:
5363/// reg(c) + reg(b) + reg({0,+,1})
5364/// reg(c) + reg({b,+,1})
5365///
5366/// Probability of not selecting
5367/// Use1 Use2 Use3
5368/// reg(a) (1/3) * 1 * 1
5369/// reg(b) 1 * (1/3) * (1/2)
5370/// reg({0,+,1}) (2/3) * (2/3) * (1/2)
5371/// reg({-1,+,1}) (2/3) * (2/3) * 1
5372/// reg({a,+,1}) (2/3) * 1 * 1
5373/// reg({b,+,1}) 1 * (2/3) * (2/3)
5374/// reg(c) 1 * 1 * 0
5375///
5376/// Now count registers number mathematical expectation for each formula:
5377/// Note that for each use we exclude probability if not selecting for the use.
5378/// For example for Use1 probability for reg(a) would be just 1 * 1 (excluding
5379/// probabilty 1/3 of not selecting for Use1).
5380/// Use1:
5381/// reg(a) + reg({0,+,1}) 1 + 1/3 -- to be deleted
5382/// reg(a) + reg({-1,+,1}) + 1 1 + 4/9 -- to be deleted
5383/// reg({a,+,1}) 1
5384/// Use2:
5385/// reg(b) + reg({0,+,1}) 1/2 + 1/3 -- to be deleted
5386/// reg(b) + reg({-1,+,1}) + 1 1/2 + 2/3 -- to be deleted
5387/// reg({b,+,1}) 2/3
5388/// Use3:
5389/// reg(c) + reg(b) + reg({0,+,1}) 1 + 1/3 + 4/9 -- to be deleted
5390/// reg(c) + reg({b,+,1}) 1 + 2/3
5391void LSRInstance::NarrowSearchSpaceByDeletingCostlyFormulas() {
5392 if (EstimateSearchSpaceComplexity() < Opts.lsr_complexity_limit)
5393 return;
5394 // Ok, we have too many of formulae on our hands to conveniently handle.
5395 // Use a rough heuristic to thin out the list.
5396
5397 // Set of Regs wich will be 100% used in final solution.
5398 // Used in each formula of a solution (in example above this is reg(c)).
5399 // We can skip them in calculations.
5400 SmallPtrSet<const SCEV *, 4> UniqRegs;
5401 LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5402
5403 // Map each register to probability of not selecting
5404 DenseMap <const SCEV *, float> RegNumMap;
5405 for (const SCEV *Reg : RegUses) {
5406 if (UniqRegs.count(Reg))
5407 continue;
5408 float PNotSel = 1;
5409 for (const LSRUse &LU : Uses) {
5410 if (!LU.Regs.count(Reg))
5411 continue;
5412 float P = LU.getNotSelectedProbability(Reg);
5413 if (P != 0.0)
5414 PNotSel *= P;
5415 else
5416 UniqRegs.insert(Reg);
5417 }
5418 RegNumMap.insert(std::make_pair(Reg, PNotSel));
5419 }
5420
5421 LLVM_DEBUG(
5422 dbgs() << "Narrowing the search space by deleting costly formulas\n");
5423
5424 // Delete formulas where registers number expectation is high.
5425 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5426 LSRUse &LU = Uses[LUIdx];
5427 // If nothing to delete - continue.
5428 if (LU.Formulae.size() < 2)
5429 continue;
5430 // This is temporary solution to test performance. Float should be
5431 // replaced with round independent type (based on integers) to avoid
5432 // different results for different target builds.
5433 float FMinRegNum = LU.Formulae[0].getNumRegs();
5434 float FMinARegNum = LU.Formulae[0].getNumRegs();
5435 size_t MinIdx = 0;
5436 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
5437 Formula &F = LU.Formulae[i];
5438 float FRegNum = 0;
5439 float FARegNum = 0;
5440 for (const SCEV *BaseReg : F.BaseRegs) {
5441 if (UniqRegs.count(BaseReg))
5442 continue;
5443 FRegNum += RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
5444 if (isa<SCEVAddRecExpr>(BaseReg))
5445 FARegNum +=
5446 RegNumMap[BaseReg] / LU.getNotSelectedProbability(BaseReg);
5447 }
5448 if (const SCEV *ScaledReg = F.ScaledReg) {
5449 if (!UniqRegs.count(ScaledReg)) {
5450 FRegNum +=
5451 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
5452 if (isa<SCEVAddRecExpr>(ScaledReg))
5453 FARegNum +=
5454 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(ScaledReg);
5455 }
5456 }
5457 if (FMinRegNum > FRegNum ||
5458 (FMinRegNum == FRegNum && FMinARegNum > FARegNum)) {
5459 FMinRegNum = FRegNum;
5460 FMinARegNum = FARegNum;
5461 MinIdx = i;
5462 }
5463 }
5464 LLVM_DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs());
5465 dbgs() << " with min reg num " << FMinRegNum << '\n');
5466 if (MinIdx != 0)
5467 std::swap(LU.Formulae[MinIdx], LU.Formulae[0]);
5468 while (LU.Formulae.size() != 1) {
5469 LLVM_DEBUG(dbgs() << " Deleting "; LU.Formulae.back().print(dbgs());
5470 dbgs() << '\n');
5471 LU.Formulae.pop_back();
5472 }
5473 LU.RecomputeRegs(LUIdx, RegUses);
5474 assert(LU.Formulae.size() == 1 && "Should be exactly 1 min regs formula");
5475 Formula &F = LU.Formulae[0];
5476 LLVM_DEBUG(dbgs() << " Leaving only "; F.print(dbgs()); dbgs() << '\n');
5477 // When we choose the formula, the regs become unique.
5478 UniqRegs.insert_range(F.BaseRegs);
5479 if (F.ScaledReg)
5480 UniqRegs.insert(F.ScaledReg);
5481 }
5482 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5483}
5484
5485// Check if Best and Reg are SCEVs separated by a constant amount C, and if so
5486// would the addressing offset +C would be legal where the negative offset -C is
5487// not.
5489 ScalarEvolution &SE, const SCEV *Best,
5490 const SCEV *Reg,
5491 MemAccessTy AccessType) {
5492 if (Best->getType() != Reg->getType() ||
5494 cast<SCEVAddRecExpr>(Best)->getLoop() !=
5495 cast<SCEVAddRecExpr>(Reg)->getLoop()))
5496 return false;
5497 std::optional<APInt> Diff = SE.computeConstantDifference(Best, Reg);
5498 if (!Diff)
5499 return false;
5500
5501 return TTI.isLegalAddressingMode(
5502 AccessType.MemTy, /*BaseGV=*/nullptr,
5503 /*BaseOffset=*/Diff->getSExtValue(),
5504 /*HasBaseReg=*/true, /*Scale=*/0, AccessType.AddrSpace) &&
5505 !TTI.isLegalAddressingMode(
5506 AccessType.MemTy, /*BaseGV=*/nullptr,
5507 /*BaseOffset=*/-Diff->getSExtValue(),
5508 /*HasBaseReg=*/true, /*Scale=*/0, AccessType.AddrSpace);
5509}
5510
5511/// Pick a register which seems likely to be profitable, and then in any use
5512/// which has any reference to that register, delete all formulae which do not
5513/// reference that register.
5514void LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() {
5515 // With all other options exhausted, loop until the system is simple
5516 // enough to handle.
5517 SmallPtrSet<const SCEV *, 4> Taken;
5518 while (EstimateSearchSpaceComplexity() >= Opts.lsr_complexity_limit) {
5519 // Ok, we have too many of formulae on our hands to conveniently handle.
5520 // Use a rough heuristic to thin out the list.
5521 LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5522
5523 // Pick the register which is used by the most LSRUses, which is likely
5524 // to be a good reuse register candidate.
5525 const SCEV *Best = nullptr;
5526 unsigned BestNum = 0;
5527 for (const SCEV *Reg : RegUses) {
5528 if (Taken.count(Reg))
5529 continue;
5530 if (!Best) {
5531 Best = Reg;
5532 BestNum = RegUses.getUsedByIndices(Reg).count();
5533 } else {
5534 unsigned Count = RegUses.getUsedByIndices(Reg).count();
5535 if (Count > BestNum) {
5536 Best = Reg;
5537 BestNum = Count;
5538 }
5539
5540 // If the scores are the same, but the Reg is simpler for the target
5541 // (for example {x,+,1} as opposed to {x+C,+,1}, where the target can
5542 // handle +C but not -C), opt for the simpler formula.
5543 if (Count == BestNum) {
5544 int LUIdx = RegUses.getUsedByIndices(Reg).find_first();
5545 if (LUIdx >= 0 && Uses[LUIdx].Kind == LSRUse::Address &&
5547 Uses[LUIdx].AccessTy)) {
5548 Best = Reg;
5549 BestNum = Count;
5550 }
5551 }
5552 }
5553 }
5554 assert(Best && "Failed to find best LSRUse candidate");
5555
5556 LLVM_DEBUG(dbgs() << "Narrowing the search space by assuming " << *Best
5557 << " will yield profitable reuse.\n");
5558 Taken.insert(Best);
5559
5560 // In any use with formulae which references this register, delete formulae
5561 // which don't reference it.
5562 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5563 LSRUse &LU = Uses[LUIdx];
5564 if (!LU.Regs.count(Best)) continue;
5565
5566 bool Any = false;
5567 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
5568 Formula &F = LU.Formulae[i];
5569 if (!F.referencesReg(Best)) {
5570 LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n');
5571 LU.DeleteFormula(F);
5572 --e;
5573 --i;
5574 Any = true;
5575 assert(e != 0 && "Use has no formulae left! Is Regs inconsistent?");
5576 continue;
5577 }
5578 }
5579
5580 if (Any)
5581 LU.RecomputeRegs(LUIdx, RegUses);
5582 }
5583
5584 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5585 }
5586}
5587
5588/// If there are an extraordinary number of formulae to choose from, use some
5589/// rough heuristics to prune down the number of formulae. This keeps the main
5590/// solver from taking an extraordinary amount of time in some worst-case
5591/// scenarios.
5592void LSRInstance::NarrowSearchSpaceUsingHeuristics() {
5593 NarrowSearchSpaceByDetectingSupersets();
5594 NarrowSearchSpaceByCollapsingUnrolledCode();
5595 NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
5596 if (Opts.lsr_filter_same_scaled_reg)
5597 NarrowSearchSpaceByFilterFormulaWithSameScaledReg();
5598 NarrowSearchSpaceByFilterPostInc();
5599 NarrowSearchSpaceByMergingUsesOutsideLoop();
5600 if (Opts.lsr_exp_narrow)
5601 NarrowSearchSpaceByDeletingCostlyFormulas();
5602 else
5603 NarrowSearchSpaceByPickingWinnerRegs();
5604}
5605
5606/// This is the recursive solver.
5607void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
5608 Cost &SolutionCost,
5609 SmallVectorImpl<const Formula *> &Workspace,
5610 const Cost &CurCost,
5611 const SmallPtrSet<const SCEV *, 16> &CurRegs,
5612 DenseSet<const SCEV *> &VisitedRegs) const {
5613 // Some ideas:
5614 // - prune more:
5615 // - use more aggressive filtering
5616 // - sort the formula so that the most profitable solutions are found first
5617 // - sort the uses too
5618 // - search faster:
5619 // - don't compute a cost, and then compare. compare while computing a cost
5620 // and bail early.
5621 // - track register sets with SmallBitVector
5622
5623 const LSRUse &LU = Uses[Workspace.size()];
5624
5625 // If this use references any register that's already a part of the
5626 // in-progress solution, consider it a requirement that a formula must
5627 // reference that register in order to be considered. This prunes out
5628 // unprofitable searching.
5629 SmallSetVector<const SCEV *, 4> ReqRegs;
5630 for (const SCEV *S : CurRegs)
5631 if (LU.Regs.count(S))
5632 ReqRegs.insert(S);
5633
5634 SmallPtrSet<const SCEV *, 16> NewRegs;
5635 Cost NewCost(Opts, L, SE, TTI, AMK);
5636 for (const Formula &F : LU.Formulae) {
5637 // Ignore formulae which may not be ideal in terms of register reuse of
5638 // ReqRegs. The formula should use all required registers before
5639 // introducing new ones.
5640 // This can sometimes (notably when trying to favour postinc) lead to
5641 // sub-optimial decisions. There it is best left to the cost modelling to
5642 // get correct.
5643 if (!(AMK & TTI::AMK_PostIndexed) || LU.Kind != LSRUse::Address) {
5644 int NumReqRegsToFind = std::min(F.getNumRegs(), ReqRegs.size());
5645 for (const SCEV *Reg : ReqRegs) {
5646 if ((F.ScaledReg && F.ScaledReg == Reg) ||
5647 is_contained(F.BaseRegs, Reg)) {
5648 --NumReqRegsToFind;
5649 if (NumReqRegsToFind == 0)
5650 break;
5651 }
5652 }
5653 if (NumReqRegsToFind != 0) {
5654 // If none of the formulae satisfied the required registers, then we could
5655 // clear ReqRegs and try again. Currently, we simply give up in this case.
5656 continue;
5657 }
5658 }
5659
5660 // Evaluate the cost of the current formula. If it's already worse than
5661 // the current best, prune the search at that point.
5662 NewCost = CurCost;
5663 NewRegs = CurRegs;
5664 NewCost.RateFormula(F, NewRegs, VisitedRegs, LU, HardwareLoopProfitable);
5665 if (NewCost.isLess(SolutionCost)) {
5666 Workspace.push_back(&F);
5667 if (Workspace.size() != Uses.size()) {
5668 SolveRecurse(Solution, SolutionCost, Workspace, NewCost,
5669 NewRegs, VisitedRegs);
5670 if (F.getNumRegs() == 1 && Workspace.size() == 1)
5671 VisitedRegs.insert(F.ScaledReg ? F.ScaledReg : F.BaseRegs[0]);
5672 } else {
5673 LLVM_DEBUG(dbgs() << "New best at "; NewCost.print(dbgs());
5674 dbgs() << ".\nRegs:\n";
5675 for (const SCEV *S : NewRegs) dbgs()
5676 << "- " << *S << "\n";
5677 dbgs() << '\n');
5678
5679 SolutionCost = NewCost;
5680 Solution = Workspace;
5681 }
5682 Workspace.pop_back();
5683 }
5684 }
5685}
5686
5687/// Choose one formula from each use. Return the results in the given Solution
5688/// vector.
5689void LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const {
5691 Cost SolutionCost(Opts, L, SE, TTI, AMK);
5692 SolutionCost.Lose();
5693 Cost CurCost(Opts, L, SE, TTI, AMK);
5694 SmallPtrSet<const SCEV *, 16> CurRegs;
5695 DenseSet<const SCEV *> VisitedRegs;
5696 Workspace.reserve(Uses.size());
5697
5698 // SolveRecurse does all the work.
5699 SolveRecurse(Solution, SolutionCost, Workspace, CurCost,
5700 CurRegs, VisitedRegs);
5701 if (Solution.empty()) {
5702 LLVM_DEBUG(dbgs() << "\nNo Satisfactory Solution\n");
5703 return;
5704 }
5705
5706 // Ok, we've now made all our decisions.
5707 LLVM_DEBUG(dbgs() << "\n"
5708 "The chosen solution requires ";
5709 SolutionCost.print(dbgs()); dbgs() << ":\n";
5710 for (size_t i = 0, e = Uses.size(); i != e; ++i) {
5711 dbgs() << " ";
5712 Uses[i].print(dbgs());
5713 dbgs() << "\n"
5714 " ";
5715 Solution[i]->print(dbgs());
5716 dbgs() << '\n';
5717 });
5718
5719 assert(Solution.size() == Uses.size() && "Malformed solution!");
5720
5721 const bool EnableDropUnprofitableSolution = valueOr(
5722 Opts.lsr_drop_solution, TTI.shouldDropLSRSolutionIfLessProfitable());
5723
5724 if (BaselineCost.isLess(SolutionCost)) {
5725 if (!EnableDropUnprofitableSolution)
5726 LLVM_DEBUG(
5727 dbgs() << "Baseline is more profitable than chosen solution, "
5728 "add option 'lsr-drop-solution' to drop LSR solution.\n");
5729 else {
5730 LLVM_DEBUG(dbgs() << "Baseline is more profitable than chosen "
5731 "solution, dropping LSR solution.\n";);
5732 Solution.clear();
5733 }
5734 }
5735}
5736
5737/// Helper for AdjustInsertPositionForExpand. Climb up the dominator tree far as
5738/// we can go while still being dominated by the input positions. This helps
5739/// canonicalize the insert position, which encourages sharing.
5741LSRInstance::HoistInsertPosition(BasicBlock::iterator IP,
5742 const SmallVectorImpl<Instruction *> &Inputs)
5743 const {
5744 Instruction *Tentative = &*IP;
5745 while (true) {
5746 bool AllDominate = true;
5747 Instruction *BetterPos = nullptr;
5748 // Don't bother attempting to insert before a catchswitch, their basic block
5749 // cannot have other non-PHI instructions.
5750 if (isa<CatchSwitchInst>(Tentative))
5751 return IP;
5752
5753 for (Instruction *Inst : Inputs) {
5754 if (Inst == Tentative || !DT.dominates(Inst, Tentative)) {
5755 AllDominate = false;
5756 break;
5757 }
5758 // Attempt to find an insert position in the middle of the block,
5759 // instead of at the end, so that it can be used for other expansions.
5760 if (Tentative->getParent() == Inst->getParent() &&
5761 (!BetterPos || !DT.dominates(Inst, BetterPos)))
5762 BetterPos = &*std::next(BasicBlock::iterator(Inst));
5763 }
5764 if (!AllDominate)
5765 break;
5766 if (BetterPos)
5767 IP = BetterPos->getIterator();
5768 else
5769 IP = Tentative->getIterator();
5770
5771 const Loop *IPLoop = LI.getLoopFor(IP->getParent());
5772 unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : 0;
5773
5774 BasicBlock *IDom;
5775 for (DomTreeNode *Rung = DT.getNode(IP->getParent()); ; ) {
5776 if (!Rung) return IP;
5777 Rung = Rung->getIDom();
5778 if (!Rung) return IP;
5779 IDom = Rung->getBlock();
5780
5781 // Don't climb into a loop though.
5782 const Loop *IDomLoop = LI.getLoopFor(IDom);
5783 unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : 0;
5784 if (IDomDepth <= IPLoopDepth &&
5785 (IDomDepth != IPLoopDepth || IDomLoop == IPLoop))
5786 break;
5787 }
5788
5789 Tentative = IDom->getTerminator();
5790 }
5791
5792 return IP;
5793}
5794
5795/// Determine an input position which will be dominated by the operands and
5796/// which will dominate the result.
5797BasicBlock::iterator LSRInstance::AdjustInsertPositionForExpand(
5798 BasicBlock::iterator LowestIP, const LSRFixup &LF, const LSRUse &LU) const {
5799 // Collect some instructions which must be dominated by the
5800 // expanding replacement. These must be dominated by any operands that
5801 // will be required in the expansion.
5802 SmallVector<Instruction *, 4> Inputs;
5803 if (Instruction *I = dyn_cast<Instruction>(LF.OperandValToReplace))
5804 Inputs.push_back(I);
5805 if (LU.Kind == LSRUse::ICmpZero)
5806 if (Instruction *I =
5807 dyn_cast<Instruction>(cast<ICmpInst>(LF.UserInst)->getOperand(1)))
5808 Inputs.push_back(I);
5809 if (LF.PostIncLoops.count(L)) {
5810 if (LF.isUseFullyOutsideLoop(L))
5811 Inputs.push_back(L->getLoopLatch()->getTerminator());
5812 else
5813 Inputs.push_back(IVIncInsertPos);
5814 }
5815 // The expansion must also be dominated by the increment positions of any
5816 // loops it for which it is using post-inc mode.
5817 for (const Loop *PIL : LF.PostIncLoops) {
5818 if (PIL == L) continue;
5819
5820 // Be dominated by the loop exit.
5821 SmallVector<BasicBlock *, 4> ExitingBlocks;
5822 PIL->getExitingBlocks(ExitingBlocks);
5823 if (!ExitingBlocks.empty()) {
5824 BasicBlock *BB = ExitingBlocks[0];
5825 for (unsigned i = 1, e = ExitingBlocks.size(); i != e; ++i)
5826 BB = DT.findNearestCommonDominator(BB, ExitingBlocks[i]);
5827 Inputs.push_back(BB->getTerminator());
5828 }
5829 }
5830
5831 assert(!isa<PHINode>(LowestIP) && !LowestIP->isEHPad() &&
5832 "Insertion point must be a normal instruction");
5833
5834 // Then, climb up the immediate dominator tree as far as we can go while
5835 // still being dominated by the input positions.
5836 BasicBlock::iterator IP = HoistInsertPosition(LowestIP, Inputs);
5837
5838 // Don't insert instructions before PHI nodes.
5839 while (isa<PHINode>(IP)) ++IP;
5840
5841 // Ignore landingpad instructions.
5842 while (IP->isEHPad()) ++IP;
5843
5844 // Set IP below instructions recently inserted by SCEVExpander. This keeps the
5845 // IP consistent across expansions and allows the previously inserted
5846 // instructions to be reused by subsequent expansion.
5847 while (Rewriter.isInsertedInstruction(&*IP) && IP != LowestIP)
5848 ++IP;
5849
5850 return IP;
5851}
5852
5853/// Emit instructions for the leading candidate expression for this LSRUse (this
5854/// is called "expanding").
5855Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF,
5856 const Formula &F, BasicBlock::iterator IP,
5857 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const {
5858 if (LU.RigidFormula)
5859 return LF.OperandValToReplace;
5860
5861 // Determine an input position which will be dominated by the operands and
5862 // which will dominate the result.
5863 IP = AdjustInsertPositionForExpand(IP, LF, LU);
5864 Rewriter.setInsertPoint(&*IP);
5865
5866 // Inform the Rewriter if we have a post-increment use, so that it can
5867 // perform an advantageous expansion.
5868 Rewriter.setPostInc(LF.PostIncLoops);
5869
5870 // This is the type that the user actually needs.
5871 Type *OpTy = LF.OperandValToReplace->getType();
5872 // This will be the type that we'll initially expand to.
5873 Type *Ty = F.getType();
5874 if (!Ty)
5875 // No type known; just expand directly to the ultimate type.
5876 Ty = OpTy;
5877 else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(OpTy))
5878 // Expand directly to the ultimate type if it's the right size.
5879 Ty = OpTy;
5880 // This is the type to do integer arithmetic in.
5881 Type *IntTy = SE.getEffectiveSCEVType(Ty);
5882 // For ICmpZero with pointer-typed operands, keep the comparison in the
5883 // integer domain to avoid generating inttoptr casts. Use IntTy (the
5884 // formula's arithmetic width) so that both icmp operands match even when
5885 // the IV is wider than the pointer.
5886 if (LU.Kind == LSRUse::ICmpZero && OpTy->isPointerTy()) {
5887 OpTy = IntTy;
5888 Ty = IntTy;
5889 }
5890
5891 // Build up a list of operands to add together to form the full base.
5893
5894 // Expand the BaseRegs portion.
5895 for (const SCEV *Reg : F.BaseRegs) {
5896 assert(!Reg->isZero() && "Zero allocated in a base register!");
5897
5898 // If we're expanding for a post-inc user, make the post-inc adjustment.
5899 Reg = denormalizeForPostIncUse(Reg, LF.PostIncLoops, SE);
5900 Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, nullptr)));
5901 }
5902
5903 // Expand the ScaledReg portion.
5904 Value *ICmpScaledV = nullptr;
5905 if (F.Scale != 0) {
5906 const SCEV *ScaledS = F.ScaledReg;
5907
5908 // If we're expanding for a post-inc user, make the post-inc adjustment.
5909 PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops);
5910 ScaledS = denormalizeForPostIncUse(ScaledS, Loops, SE);
5911
5912 if (LU.Kind == LSRUse::ICmpZero) {
5913 // Expand ScaleReg as if it was part of the base regs.
5914 if (F.Scale == 1)
5915 Ops.push_back(
5916 SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr)));
5917 else {
5918 // An interesting way of "folding" with an icmp is to use a negated
5919 // scale, which we'll implement by inserting it into the other operand
5920 // of the icmp.
5921 assert(F.Scale == -1 &&
5922 "The only scale supported by ICmpZero uses is -1!");
5923 ICmpScaledV = Rewriter.expandCodeFor(ScaledS, nullptr);
5924 }
5925 } else {
5926 // Otherwise just expand the scaled register and an explicit scale,
5927 // which is expected to be matched as part of the address.
5928
5929 // Flush the operand list to suppress SCEVExpander hoisting address modes.
5930 // Unless the addressing mode will not be folded.
5931 if (!Ops.empty() && LU.Kind == LSRUse::Address &&
5932 isAMCompletelyFolded(TTI, LU, F)) {
5933 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), nullptr);
5934 Ops.clear();
5935 Ops.push_back(SE.getUnknown(FullV));
5936 }
5937 ScaledS = SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr));
5938 if (F.Scale != 1)
5939 ScaledS =
5940 SE.getMulExpr(ScaledS, SE.getConstant(ScaledS->getType(), F.Scale));
5941 Ops.push_back(ScaledS);
5942 }
5943 }
5944
5945 // Expand the GV portion.
5946 if (F.BaseGV) {
5947 // Flush the operand list to suppress SCEVExpander hoisting.
5948 if (!Ops.empty()) {
5949 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), IntTy);
5950 Ops.clear();
5951 Ops.push_back(SE.getUnknown(FullV));
5952 }
5953 Ops.push_back(SE.getUnknown(F.BaseGV));
5954 }
5955
5956 // Flush the operand list to suppress SCEVExpander hoisting of both folded and
5957 // unfolded offsets. LSR assumes they both live next to their uses.
5958 if (!Ops.empty()) {
5959 Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty);
5960 Ops.clear();
5961 Ops.push_back(SE.getUnknown(FullV));
5962 }
5963
5964 // FIXME: Are we sure we won't get a mismatch here? Is there a way to bail
5965 // out at this point, or should we generate a SCEV adding together mixed
5966 // offsets?
5967 assert(F.BaseOffset.isCompatibleImmediate(LF.Offset) &&
5968 "Expanding mismatched offsets\n");
5969 // Expand the immediate portion.
5970 Immediate Offset = F.BaseOffset.addUnsigned(LF.Offset);
5971 if (Offset.isNonZero()) {
5972 if (LU.Kind == LSRUse::ICmpZero) {
5973 // The other interesting way of "folding" with an ICmpZero is to use a
5974 // negated immediate.
5975 if (!ICmpScaledV) {
5976 // TODO: Avoid implicit trunc?
5977 // See https://github.com/llvm/llvm-project/issues/112510.
5978 ICmpScaledV = ConstantInt::getSigned(
5979 IntTy, -(uint64_t)Offset.getFixedValue(), /*ImplicitTrunc=*/true);
5980 } else {
5981 Ops.push_back(SE.getUnknown(ICmpScaledV));
5982 ICmpScaledV = ConstantInt::getSigned(IntTy, Offset.getFixedValue(),
5983 /*ImplicitTrunc=*/true);
5984 }
5985 } else {
5986 // Just add the immediate values. These again are expected to be matched
5987 // as part of the address.
5988 Ops.push_back(Offset.getUnknownSCEV(SE, IntTy));
5989 }
5990 }
5991
5992 // Expand the unfolded offset portion.
5993 Immediate UnfoldedOffset = F.UnfoldedOffset;
5994 if (UnfoldedOffset.isNonZero()) {
5995 // Just add the immediate values.
5996 Ops.push_back(UnfoldedOffset.getUnknownSCEV(SE, IntTy));
5997 }
5998
5999 // Emit instructions summing all the operands.
6000 const SCEV *FullS =
6001 Ops.empty() ? SE.getConstant(IntTy, 0) : SE.getAddExpr(Ops).getPointer();
6002 Value *FullV = Rewriter.expandCodeFor(FullS, Ty);
6003
6004 // We're done expanding now, so reset the rewriter.
6005 Rewriter.clearPostInc();
6006
6007 // An ICmpZero Formula represents an ICmp which we're handling as a
6008 // comparison against zero. Now that we've expanded an expression for that
6009 // form, update the ICmp's other operand.
6010 if (LU.Kind == LSRUse::ICmpZero) {
6011 ICmpInst *CI = cast<ICmpInst>(LF.UserInst);
6012 if (auto *OperandIsInstr = dyn_cast<Instruction>(CI->getOperand(1)))
6013 DeadInsts.emplace_back(OperandIsInstr);
6014 assert(!F.BaseGV && "ICmp does not support folding a global value and "
6015 "a scale at the same time!");
6016 if (F.Scale == -1) {
6017 if (ICmpScaledV->getType() != OpTy) {
6019 CastInst::getCastOpcode(ICmpScaledV, false, OpTy, false),
6020 ICmpScaledV, OpTy, "tmp", CI->getIterator());
6021 ICmpScaledV = Cast;
6022 }
6023 CI->setOperand(1, ICmpScaledV);
6024 } else {
6025 // A scale of 1 means that the scale has been expanded as part of the
6026 // base regs.
6027 assert((F.Scale == 0 || F.Scale == 1) &&
6028 "ICmp does not support folding a global value and "
6029 "a scale at the same time!");
6030 // TODO: Avoid implicit trunc?
6031 // See https://github.com/llvm/llvm-project/issues/112510.
6033 -(uint64_t)Offset.getFixedValue(),
6034 /*ImplicitTrunc=*/true);
6035 if (C->getType() != OpTy) {
6037 CastInst::getCastOpcode(C, false, OpTy, false), C, OpTy,
6038 CI->getDataLayout());
6039 assert(C && "Cast of ConstantInt should have folded");
6040 }
6041
6042 CI->setOperand(1, C);
6043 }
6044 }
6045
6046 return FullV;
6047}
6048
6049/// Helper for Rewrite. PHI nodes are special because the use of their operands
6050/// effectively happens in their predecessor blocks, so the expression may need
6051/// to be expanded in multiple places.
6052void LSRInstance::RewriteForPHI(PHINode *PN, const LSRUse &LU,
6053 const LSRFixup &LF, const Formula &F,
6054 SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
6055 DenseMap<BasicBlock *, Value *> Inserted;
6056
6057 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
6058 if (PN->getIncomingValue(i) == LF.OperandValToReplace) {
6059 bool needUpdateFixups = false;
6060 BasicBlock *BB = PN->getIncomingBlock(i);
6061
6062 // If this is a critical edge, split the edge so that we do not insert
6063 // the code on all predecessor/successor paths. We do this unless this
6064 // is the canonical backedge for this loop, which complicates post-inc
6065 // users.
6066 if (e != 1 && BB->getTerminator()->getNumSuccessors() > 1 &&
6069 BasicBlock *Parent = PN->getParent();
6070 Loop *PNLoop = LI.getLoopFor(Parent);
6071 if (!PNLoop || Parent != PNLoop->getHeader()) {
6072 // Split the critical edge.
6073 BasicBlock *NewBB = nullptr;
6074 if (!Parent->isLandingPad()) {
6075 CriticalEdgeSplittingOptions SplitOptions(&DT, &LI, MSSAU);
6076 SplitOptions =
6077 SplitOptions.setMergeIdenticalEdges().setKeepOneInputPHIs();
6078 if (ShouldPreserveLCSSA)
6079 SplitOptions = SplitOptions.setPreserveLCSSA();
6080 NewBB = SplitCriticalEdge(BB, Parent, SplitOptions);
6081 } else {
6083 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
6084 SplitLandingPadPredecessors(Parent, BB, "", "", NewBBs, &DTU, &LI,
6085 MSSAU, ShouldPreserveLCSSA);
6086 NewBB = NewBBs[0];
6087 }
6088 // If NewBB==NULL, then SplitCriticalEdge refused to split because all
6089 // phi predecessors are identical. The simple thing to do is skip
6090 // splitting in this case rather than complicate the API.
6091 if (NewBB) {
6092 // If PN is outside of the loop and BB is in the loop, we want to
6093 // move the block to be immediately before the PHI block, not
6094 // immediately after BB.
6095 if (L->contains(BB) && !L->contains(PN))
6096 NewBB->moveBefore(PN->getParent());
6097
6098 // Splitting the edge can reduce the number of PHI entries we have.
6099 e = PN->getNumIncomingValues();
6100 BB = NewBB;
6101 i = PN->getBasicBlockIndex(BB);
6102
6103 needUpdateFixups = true;
6104
6105 if (auto *OldIncomingInst =
6107 DeadInsts.emplace_back(OldIncomingInst);
6108 }
6109 }
6110 }
6111
6112 std::pair<DenseMap<BasicBlock *, Value *>::iterator, bool> Pair =
6113 Inserted.try_emplace(BB);
6114 if (!Pair.second)
6115 PN->setIncomingValue(i, Pair.first->second);
6116 else {
6117 Value *FullV =
6118 Expand(LU, LF, F, BB->getTerminator()->getIterator(), DeadInsts);
6119
6120 // If this is reuse-by-noop-cast, insert the noop cast.
6121 Type *OpTy = LF.OperandValToReplace->getType();
6122 if (FullV->getType() != OpTy)
6123 FullV = CastInst::Create(
6124 CastInst::getCastOpcode(FullV, false, OpTy, false), FullV,
6125 LF.OperandValToReplace->getType(), "tmp",
6126 BB->getTerminator()->getIterator());
6127
6128 // If the incoming block for this value is not in the loop, it means the
6129 // current PHI is not in a loop exit, so we must create a LCSSA PHI for
6130 // the inserted value.
6131 if (auto *I = dyn_cast<Instruction>(FullV))
6132 if (L->contains(I) && !L->contains(BB))
6133 InsertedNonLCSSAInsts.insert(I);
6134
6135 PN->setIncomingValue(i, FullV);
6136 Pair.first->second = FullV;
6137 }
6138
6139 // If LSR splits critical edge and phi node has other pending
6140 // fixup operands, we need to update those pending fixups. Otherwise
6141 // formulae will not be implemented completely and some instructions
6142 // will not be eliminated.
6143 if (needUpdateFixups) {
6144 for (LSRUse &LU : Uses)
6145 for (LSRFixup &Fixup : LU.Fixups)
6146 // If fixup is supposed to rewrite some operand in the phi
6147 // that was just updated, it may be already moved to
6148 // another phi node. Such fixup requires update.
6149 if (Fixup.UserInst == PN) {
6150 // Check if the operand we try to replace still exists in the
6151 // original phi.
6152 bool foundInOriginalPHI = false;
6153 for (const auto &val : PN->incoming_values())
6154 if (val == Fixup.OperandValToReplace) {
6155 foundInOriginalPHI = true;
6156 break;
6157 }
6158
6159 // If fixup operand found in original PHI - nothing to do.
6160 if (foundInOriginalPHI)
6161 continue;
6162
6163 // Otherwise it might be moved to another PHI and requires update.
6164 // If fixup operand not found in any of the incoming blocks that
6165 // means we have already rewritten it - nothing to do.
6166 for (const auto &Block : PN->blocks())
6167 for (BasicBlock::iterator I = Block->begin(); isa<PHINode>(I);
6168 ++I) {
6169 PHINode *NewPN = cast<PHINode>(I);
6170 for (const auto &val : NewPN->incoming_values())
6171 if (val == Fixup.OperandValToReplace)
6172 Fixup.UserInst = NewPN;
6173 }
6174 }
6175 }
6176 }
6177}
6178
6179/// Emit instructions for the leading candidate expression for this LSRUse (this
6180/// is called "expanding"), and update the UserInst to reference the newly
6181/// expanded value.
6182void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF,
6183 const Formula &F,
6184 SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
6185 // First, find an insertion point that dominates UserInst. For PHI nodes,
6186 // find the nearest block which dominates all the relevant uses.
6187 if (PHINode *PN = dyn_cast<PHINode>(LF.UserInst)) {
6188 RewriteForPHI(PN, LU, LF, F, DeadInsts);
6189 } else {
6190 Value *FullV = Expand(LU, LF, F, LF.UserInst->getIterator(), DeadInsts);
6191
6192 // If this is reuse-by-noop-cast, insert the noop cast.
6193 // For ICmpZero with pointer operands, Expand() already set both operands
6194 // in integer domain, so no cast is needed here.
6195 Type *OpTy = LF.OperandValToReplace->getType();
6196 if (FullV->getType() != OpTy &&
6197 !(LU.Kind == LSRUse::ICmpZero && OpTy->isPointerTy())) {
6198 Instruction *Cast =
6199 CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false),
6200 FullV, OpTy, "tmp", LF.UserInst->getIterator());
6201 FullV = Cast;
6202 }
6203
6204 // Update the user. ICmpZero is handled specially here (for now) because
6205 // Expand may have updated one of the operands of the icmp already, and
6206 // its new value may happen to be equal to LF.OperandValToReplace, in
6207 // which case doing replaceUsesOfWith leads to replacing both operands
6208 // with the same value. TODO: Reorganize this.
6209 if (LU.Kind == LSRUse::ICmpZero)
6210 LF.UserInst->setOperand(0, FullV);
6211 else
6212 LF.UserInst->replaceUsesOfWith(LF.OperandValToReplace, FullV);
6213 }
6214
6215 if (auto *OperandIsInstr = dyn_cast<Instruction>(LF.OperandValToReplace))
6216 DeadInsts.emplace_back(OperandIsInstr);
6217}
6218
6219// Determine where to insert the transformed IV increment instruction for this
6220// fixup. By default this is the default insert position, but if this is a
6221// postincrement opportunity then we try to insert it in the same block as the
6222// fixup user instruction, as this is needed for a postincrement instruction to
6223// be generated.
6225 const LSRFixup &Fixup, const LSRUse &LU,
6226 Instruction *IVIncInsertPos,
6227 DominatorTree &DT) {
6228 // Only address uses can be postincremented
6229 if (LU.Kind != LSRUse::Address)
6230 return IVIncInsertPos;
6231
6232 // Don't try to postincrement if it's not legal
6233 Instruction *I = Fixup.UserInst;
6234 Type *Ty = I->getType();
6235 if (!(isa<LoadInst>(I) && TTI.isIndexedLoadLegal(TTI.MIM_PostInc, Ty)) &&
6236 !(isa<StoreInst>(I) && TTI.isIndexedStoreLegal(TTI.MIM_PostInc, Ty)))
6237 return IVIncInsertPos;
6238
6239 // It's only legal to hoist to the user block if it dominates the default
6240 // insert position.
6241 BasicBlock *HoistBlock = I->getParent();
6242 BasicBlock *IVIncBlock = IVIncInsertPos->getParent();
6243 if (!DT.dominates(I, IVIncBlock))
6244 return IVIncInsertPos;
6245
6246 return HoistBlock->getTerminator();
6247}
6248
6249/// Rewrite all the fixup locations with new values, following the chosen
6250/// solution.
6251void LSRInstance::ImplementSolution(
6252 const SmallVectorImpl<const Formula *> &Solution) {
6253 // Keep track of instructions we may have made dead, so that
6254 // we can remove them after we are done working.
6256
6257 // Mark phi nodes that terminate chains so the expander tries to reuse them.
6258 for (const IVChain &Chain : IVChainVec) {
6259 if (PHINode *PN = dyn_cast<PHINode>(Chain.tailUserInst()))
6260 Rewriter.setChainedPhi(PN);
6261 }
6262
6263 // Expand the new value definitions and update the users.
6264 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx)
6265 for (const LSRFixup &Fixup : Uses[LUIdx].Fixups) {
6266 Instruction *InsertPos =
6267 getFixupInsertPos(TTI, Fixup, Uses[LUIdx], IVIncInsertPos, DT);
6268 Rewriter.setIVIncInsertPos(L, InsertPos);
6269 Rewrite(Uses[LUIdx], Fixup, *Solution[LUIdx], DeadInsts);
6270 Changed = true;
6271 }
6272
6273 auto InsertedInsts = InsertedNonLCSSAInsts.takeVector();
6274 formLCSSAForInstructions(InsertedInsts, DT, LI, &SE);
6275
6276 for (const IVChain &Chain : IVChainVec) {
6277 GenerateIVChain(Chain, DeadInsts);
6278 Changed = true;
6279 }
6280
6281 for (const WeakVH &IV : Rewriter.getInsertedIVs())
6282 if (IV && dyn_cast<Instruction>(&*IV)->getParent())
6283 ScalarEvolutionIVs.push_back(IV);
6284
6285 // Clean up after ourselves. This must be done before deleting any
6286 // instructions.
6287 Rewriter.clear();
6288
6290 &TLI, MSSAU);
6291
6292 // In our cost analysis above, we assume that each addrec consumes exactly
6293 // one register, and arrange to have increments inserted just before the
6294 // latch to maximimize the chance this is true. However, if we reused
6295 // existing IVs, we now need to move the increments to match our
6296 // expectations. Otherwise, our cost modeling results in us having a
6297 // chosen a non-optimal result for the actual schedule. (And yes, this
6298 // scheduling decision does impact later codegen.)
6299 for (PHINode &PN : L->getHeader()->phis()) {
6300 BinaryOperator *BO = nullptr;
6301 Value *Start = nullptr, *Step = nullptr;
6302 if (!matchSimpleRecurrence(&PN, BO, Start, Step))
6303 continue;
6304
6305 switch (BO->getOpcode()) {
6306 case Instruction::Sub:
6307 if (BO->getOperand(0) != &PN)
6308 // sub is non-commutative - match handling elsewhere in LSR
6309 continue;
6310 break;
6311 case Instruction::Add:
6312 break;
6313 default:
6314 continue;
6315 };
6316
6317 if (!isa<Constant>(Step))
6318 // If not a constant step, might increase register pressure
6319 // (We assume constants have been canonicalized to RHS)
6320 continue;
6321
6322 if (BO->getParent() == IVIncInsertPos->getParent())
6323 // Only bother moving across blocks. Isel can handle block local case.
6324 continue;
6325
6326 // Can we legally schedule inc at the desired point?
6327 if (!llvm::all_of(BO->uses(),
6328 [&](Use &U) {return DT.dominates(IVIncInsertPos, U);}))
6329 continue;
6330 BO->moveBefore(IVIncInsertPos->getIterator());
6331 Changed = true;
6332 }
6333
6334
6335}
6336
6337LSRInstance::LSRInstance(const ScalarOptions &Opts, Loop *L, IVUsers &IU,
6338 ScalarEvolution &SE, DominatorTree &DT, LoopInfo &LI,
6339 const TargetTransformInfo &TTI, AssumptionCache &AC,
6340 TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU,
6341 bool PreserveLCSSA)
6342 : Opts(Opts), IU(IU), SE(SE), DT(DT), LI(LI), AC(AC), TLI(TLI), TTI(TTI),
6343 L(L), MSSAU(MSSAU), AMK(Opts.lsr_preferred_addressing_mode.value_or(
6344 TTI.getPreferredAddressingMode(L, &SE))),
6345 Rewriter(SE, "lsr", PreserveLCSSA), ShouldPreserveLCSSA(PreserveLCSSA),
6346 BaselineCost(Opts, L, SE, TTI, AMK) {
6347 // If LoopSimplify form is not available, stay out of trouble.
6348 if (!L->isLoopSimplifyForm())
6349 return;
6350
6351 // If there's no interesting work to be done, bail early.
6352 if (IU.empty()) return;
6353
6354 // If there's too much analysis to be done, bail early. We won't be able to
6355 // model the problem anyway.
6356 unsigned NumUsers = 0;
6357 for (const IVStrideUse &U : IU) {
6358 if (++NumUsers > MaxIVUsers) {
6359 (void)U;
6360 LLVM_DEBUG(dbgs() << "LSR skipping loop, too many IV Users in " << U
6361 << "\n");
6362 return;
6363 }
6364 // Bail out if we have a PHI on an EHPad that gets a value from a
6365 // CatchSwitchInst. Because the CatchSwitchInst cannot be split, there is
6366 // no good place to stick any instructions.
6367 if (auto *PN = dyn_cast<PHINode>(U.getUser())) {
6368 auto FirstNonPHI = PN->getParent()->getFirstNonPHIIt();
6369 if (isa<FuncletPadInst>(FirstNonPHI) ||
6370 isa<CatchSwitchInst>(FirstNonPHI))
6371 for (BasicBlock *PredBB : PN->blocks())
6372 if (isa<CatchSwitchInst>(PredBB->getFirstNonPHIIt()))
6373 return;
6374 }
6375 }
6376
6377 LLVM_DEBUG(dbgs() << "\nLSR on loop ";
6378 L->getHeader()->printAsOperand(dbgs(), /*PrintType=*/false);
6379 dbgs() << ":\n");
6380
6381 // Check if we expect this loop to use a hardware loop instruction, which will
6382 // be used when calculating the costs of formulas.
6383 HardwareLoopInfo HWLoopInfo(L);
6384 HardwareLoopProfitable =
6385 TTI.isHardwareLoopProfitable(L, SE, AC, &TLI, HWLoopInfo);
6386
6387 // Configure SCEVExpander already now, so the correct mode is used for
6388 // isSafeToExpand() checks.
6389#if LLVM_ENABLE_ABI_BREAKING_CHECKS
6390 Rewriter.setDebugType(DEBUG_TYPE);
6391#endif
6392 Rewriter.disableCanonicalMode();
6393 Rewriter.enableLSRMode();
6394
6395 // First, perform some low-level loop optimizations.
6396 OptimizeShadowIV();
6397 OptimizeLoopTermCond();
6398
6399 // If loop preparation eliminates all interesting IV users, bail.
6400 if (IU.empty()) return;
6401
6402 // Skip nested loops until we can model them better with formulae.
6403 if (!L->isInnermost()) {
6404 LLVM_DEBUG(dbgs() << "LSR skipping outer loop " << *L << "\n");
6405 return;
6406 }
6407
6408 // Start collecting data and preparing for the solver.
6409 // If number of registers is not the major cost, we cannot benefit from the
6410 // current profitable chain optimization which is based on number of
6411 // registers.
6412 // FIXME: add profitable chain optimization for other kinds major cost, for
6413 // example number of instructions.
6414 if (TTI.isNumRegsMajorCostOfLSR() || stressIVChain(Opts))
6415 CollectChains();
6416 CollectInterestingTypesAndFactors();
6417 CollectFixupsAndInitialFormulae();
6418 CollectLoopInvariantFixupsAndFormulae();
6419
6420 if (Uses.empty())
6421 return;
6422
6423 LLVM_DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n";
6424 print_uses(dbgs()));
6425 LLVM_DEBUG(dbgs() << "The baseline solution requires ";
6426 BaselineCost.print(dbgs()); dbgs() << "\n");
6427
6428 // Now use the reuse data to generate a bunch of interesting ways
6429 // to formulate the values needed for the uses.
6430 GenerateAllReuseFormulae();
6431
6432 FilterOutUndesirableDedicatedRegisters();
6433 NarrowSearchSpaceUsingHeuristics();
6434
6436 Solve(Solution);
6437
6438 // Release memory that is no longer needed.
6439 Factors.clear();
6440 Types.clear();
6441 RegUses.clear();
6442
6443 if (Solution.empty())
6444 return;
6445
6446#ifndef NDEBUG
6447 // Formulae should be legal.
6448 for (const LSRUse &LU : Uses) {
6449 for (const Formula &F : LU.Formulae)
6450 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
6451 F) && "Illegal formula generated!");
6452 };
6453#endif
6454
6455 // Now that we've decided what we want, make it so.
6456 ImplementSolution(Solution);
6457}
6458
6459#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
6460void LSRInstance::print_factors_and_types(raw_ostream &OS) const {
6461 if (Factors.empty() && Types.empty()) return;
6462
6463 OS << "LSR has identified the following interesting factors and types: ";
6464 ListSeparator LS;
6465
6466 for (int64_t Factor : Factors)
6467 OS << LS << '*' << Factor;
6468
6469 for (Type *Ty : Types)
6470 OS << LS << '(' << *Ty << ')';
6471 OS << '\n';
6472}
6473
6474void LSRInstance::print_fixups(raw_ostream &OS) const {
6475 OS << "LSR is examining the following fixup sites:\n";
6476 for (const LSRUse &LU : Uses)
6477 for (const LSRFixup &LF : LU.Fixups) {
6478 dbgs() << " ";
6479 LF.print(OS);
6480 OS << '\n';
6481 }
6482}
6483
6484void LSRInstance::print_uses(raw_ostream &OS) const {
6485 OS << "LSR is examining the following uses:\n";
6486 for (const LSRUse &LU : Uses) {
6487 dbgs() << " ";
6488 LU.print(OS);
6489 OS << '\n';
6490 for (const Formula &F : LU.Formulae) {
6491 OS << " ";
6492 F.print(OS);
6493 OS << '\n';
6494 }
6495 }
6496}
6497
6498void LSRInstance::print(raw_ostream &OS) const {
6499 print_factors_and_types(OS);
6500 print_fixups(OS);
6501 print_uses(OS);
6502}
6503
6504LLVM_DUMP_METHOD void LSRInstance::dump() const {
6505 print(errs()); errs() << '\n';
6506}
6507#endif
6508
6509namespace {
6510
6511class LoopStrengthReduce : public LoopPass {
6512public:
6513 static char ID; // Pass ID, replacement for typeid
6514
6515 LoopStrengthReduce();
6516
6517private:
6518 bool runOnLoop(Loop *L, LPPassManager &LPM) override;
6519 void getAnalysisUsage(AnalysisUsage &AU) const override;
6520};
6521
6522} // end anonymous namespace
6523
6524LoopStrengthReduce::LoopStrengthReduce() : LoopPass(ID) {
6526}
6527
6528void LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const {
6529 // We split critical edges, so we change the CFG. However, we do update
6530 // many analyses if they are around.
6532
6533 AU.addRequired<LoopInfoWrapperPass>();
6534 AU.addPreserved<LoopInfoWrapperPass>();
6536 AU.addRequired<DominatorTreeWrapperPass>();
6537 AU.addPreserved<DominatorTreeWrapperPass>();
6538 AU.addRequired<ScalarEvolutionWrapperPass>();
6539 AU.addPreserved<ScalarEvolutionWrapperPass>();
6540 AU.addRequired<AssumptionCacheTracker>();
6541 AU.addRequired<TargetLibraryInfoWrapperPass>();
6542 // Requiring LoopSimplify a second time here prevents IVUsers from running
6543 // twice, since LoopSimplify was invalidated by running ScalarEvolution.
6545 AU.addRequired<IVUsersWrapperPass>();
6546 AU.addPreserved<IVUsersWrapperPass>();
6547 AU.addRequired<TargetTransformInfoWrapperPass>();
6548 AU.addPreserved<MemorySSAWrapperPass>();
6549}
6550
6551namespace {
6552
6553/// Enables more convenient iteration over a DWARF expression vector.
6555ToDwarfOpIter(SmallVectorImpl<uint64_t> &Expr) {
6556 llvm::DIExpression::expr_op_iterator Begin =
6557 llvm::DIExpression::expr_op_iterator(Expr.begin());
6558 llvm::DIExpression::expr_op_iterator End =
6559 llvm::DIExpression::expr_op_iterator(Expr.end());
6560 return {Begin, End};
6561}
6562
6563struct SCEVDbgValueBuilder {
6564 SCEVDbgValueBuilder() = default;
6565 SCEVDbgValueBuilder(const SCEVDbgValueBuilder &Base) { clone(Base); }
6566
6567 void clone(const SCEVDbgValueBuilder &Base) {
6568 LocationOps = Base.LocationOps;
6569 Expr = Base.Expr;
6570 }
6571
6572 void clear() {
6573 LocationOps.clear();
6574 Expr.clear();
6575 }
6576
6577 /// The DIExpression as we translate the SCEV.
6579 /// The location ops of the DIExpression.
6580 SmallVector<Value *, 2> LocationOps;
6581
6582 void pushOperator(uint64_t Op) { Expr.push_back(Op); }
6583 void pushUInt(uint64_t Operand) { Expr.push_back(Operand); }
6584
6585 /// Add a DW_OP_LLVM_arg to the expression, followed by the index of the value
6586 /// in the set of values referenced by the expression.
6587 void pushLocation(llvm::Value *V) {
6589 auto *It = llvm::find(LocationOps, V);
6590 unsigned ArgIndex = 0;
6591 if (It != LocationOps.end()) {
6592 ArgIndex = std::distance(LocationOps.begin(), It);
6593 } else {
6594 ArgIndex = LocationOps.size();
6595 LocationOps.push_back(V);
6596 }
6597 Expr.push_back(ArgIndex);
6598 }
6599
6600 void pushValue(const SCEVUnknown *U) {
6601 llvm::Value *V = cast<SCEVUnknown>(U)->getValue();
6602 pushLocation(V);
6603 }
6604
6605 bool pushConst(const SCEVConstant *C) {
6606 if (C->getAPInt().getSignificantBits() > 64)
6607 return false;
6608 Expr.push_back(llvm::dwarf::DW_OP_consts);
6609 Expr.push_back(C->getAPInt().getSExtValue());
6610 return true;
6611 }
6612
6613 // Iterating the expression as DWARF ops is convenient when updating
6614 // DWARF_OP_LLVM_args.
6616 return ToDwarfOpIter(Expr);
6617 }
6618
6619 /// Several SCEV types are sequences of the same arithmetic operator applied
6620 /// to constants and values that may be extended or truncated.
6621 bool pushArithmeticExpr(const llvm::SCEVCommutativeExpr *CommExpr,
6622 uint64_t DwarfOp) {
6623 assert((isa<llvm::SCEVAddExpr>(CommExpr) || isa<SCEVMulExpr>(CommExpr)) &&
6624 "Expected arithmetic SCEV type");
6625 bool Success = true;
6626 unsigned EmitOperator = 0;
6627 for (const auto &Op : CommExpr->operands()) {
6628 Success &= pushSCEV(Op);
6629
6630 if (EmitOperator >= 1)
6631 pushOperator(DwarfOp);
6632 ++EmitOperator;
6633 }
6634 return Success;
6635 }
6636
6637 // TODO: Identify and omit noop casts.
6638 bool pushCast(const llvm::SCEVCastExpr *C, bool IsSigned) {
6639 const llvm::SCEV *Inner = C->getOperand(0);
6640 const llvm::Type *Type = C->getType();
6641 uint64_t ToWidth = Type->getIntegerBitWidth();
6642 bool Success = pushSCEV(Inner);
6643 uint64_t CastOps[] = {dwarf::DW_OP_LLVM_convert, ToWidth,
6644 IsSigned ? llvm::dwarf::DW_ATE_signed
6645 : llvm::dwarf::DW_ATE_unsigned};
6646 for (const auto &Op : CastOps)
6647 pushOperator(Op);
6648 return Success;
6649 }
6650
6651 // TODO: MinMax - although these haven't been encountered in the test suite.
6652 bool pushSCEV(const llvm::SCEV *S) {
6653 bool Success = true;
6654 if (const SCEVConstant *StartInt = dyn_cast<SCEVConstant>(S)) {
6655 Success &= pushConst(StartInt);
6656
6657 } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
6658 if (!U->getValue())
6659 return false;
6660 pushLocation(U->getValue());
6661
6662 } else if (const SCEVMulExpr *MulRec = dyn_cast<SCEVMulExpr>(S)) {
6663 Success &= pushArithmeticExpr(MulRec, llvm::dwarf::DW_OP_mul);
6664
6665 } else if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) {
6666 Success &= pushSCEV(UDiv->getLHS());
6667 Success &= pushSCEV(UDiv->getRHS());
6668 pushOperator(llvm::dwarf::DW_OP_div);
6669
6670 } else if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(S)) {
6671 // Assert if a new and unknown SCEVCastEXpr type is encountered.
6674 "Unexpected cast type in SCEV.");
6675 Success &= pushCast(Cast, (isa<SCEVSignExtendExpr>(Cast)));
6676
6677 } else if (const SCEVAddExpr *AddExpr = dyn_cast<SCEVAddExpr>(S)) {
6678 Success &= pushArithmeticExpr(AddExpr, llvm::dwarf::DW_OP_plus);
6679
6680 } else if (isa<SCEVAddRecExpr>(S)) {
6681 // Nested SCEVAddRecExpr are generated by nested loops and are currently
6682 // unsupported.
6683 return false;
6684
6685 } else {
6686 return false;
6687 }
6688 return Success;
6689 }
6690
6691 /// Return true if the combination of arithmetic operator and underlying
6692 /// SCEV constant value is an identity function.
6693 bool isIdentityFunction(uint64_t Op, const SCEV *S) {
6694 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) {
6695 if (C->getAPInt().getSignificantBits() > 64)
6696 return false;
6697 int64_t I = C->getAPInt().getSExtValue();
6698 switch (Op) {
6699 case llvm::dwarf::DW_OP_plus:
6700 case llvm::dwarf::DW_OP_minus:
6701 return I == 0;
6702 case llvm::dwarf::DW_OP_mul:
6703 case llvm::dwarf::DW_OP_div:
6704 return I == 1;
6705 }
6706 }
6707 return false;
6708 }
6709
6710 /// Convert a SCEV of a value to a DIExpression that is pushed onto the
6711 /// builder's expression stack. The stack should already contain an
6712 /// expression for the iteration count, so that it can be multiplied by
6713 /// the stride and added to the start.
6714 /// Components of the expression are omitted if they are an identity function.
6715 /// Chain (non-affine) SCEVs are not supported.
6716 bool SCEVToValueExpr(const llvm::SCEVAddRecExpr &SAR, ScalarEvolution &SE) {
6717 assert(SAR.isAffine() && "Expected affine SCEV");
6718 const SCEV *Start = SAR.getStart();
6719 const SCEV *Stride = SAR.getStepRecurrence(SE);
6720
6721 // Skip pushing arithmetic noops.
6722 if (!isIdentityFunction(llvm::dwarf::DW_OP_mul, Stride)) {
6723 if (!pushSCEV(Stride))
6724 return false;
6725 pushOperator(llvm::dwarf::DW_OP_mul);
6726 }
6727 if (!isIdentityFunction(llvm::dwarf::DW_OP_plus, Start)) {
6728 if (!pushSCEV(Start))
6729 return false;
6730 pushOperator(llvm::dwarf::DW_OP_plus);
6731 }
6732 return true;
6733 }
6734
6735 /// Create an expression that is an offset from a value (usually the IV).
6736 void createOffsetExpr(int64_t Offset, Value *OffsetValue) {
6737 pushLocation(OffsetValue);
6739 LLVM_DEBUG(
6740 dbgs() << "scev-salvage: Generated IV offset expression. Offset: "
6741 << std::to_string(Offset) << "\n");
6742 }
6743
6744 /// Combine a translation of the SCEV and the IV to create an expression that
6745 /// recovers a location's value.
6746 /// returns true if an expression was created.
6747 bool createIterCountExpr(const SCEV *S,
6748 const SCEVDbgValueBuilder &IterationCount,
6749 ScalarEvolution &SE) {
6750 // SCEVs for SSA values are most frquently of the form
6751 // {start,+,stride}, but sometimes they are ({start,+,stride} + %a + ..).
6752 // This is because %a is a PHI node that is not the IV. However, these
6753 // SCEVs have not been observed to result in debuginfo-lossy optimisations,
6754 // so its not expected this point will be reached.
6755 if (!isa<SCEVAddRecExpr>(S))
6756 return false;
6757
6758 LLVM_DEBUG(dbgs() << "scev-salvage: Location to salvage SCEV: " << *S
6759 << '\n');
6760
6761 const auto *Rec = cast<SCEVAddRecExpr>(S);
6762 if (!Rec->isAffine())
6763 return false;
6764
6766 return false;
6767
6768 // Initialise a new builder with the iteration count expression. In
6769 // combination with the value's SCEV this enables recovery.
6770 clone(IterationCount);
6771 if (!SCEVToValueExpr(*Rec, SE))
6772 return false;
6773
6774 return true;
6775 }
6776
6777 /// Convert a SCEV of a value to a DIExpression that is pushed onto the
6778 /// builder's expression stack. The stack should already contain an
6779 /// expression for the iteration count, so that it can be multiplied by
6780 /// the stride and added to the start.
6781 /// Components of the expression are omitted if they are an identity function.
6782 bool SCEVToIterCountExpr(const llvm::SCEVAddRecExpr &SAR,
6783 ScalarEvolution &SE) {
6784 assert(SAR.isAffine() && "Expected affine SCEV");
6785 const SCEV *Start = SAR.getStart();
6786 const SCEV *Stride = SAR.getStepRecurrence(SE);
6787
6788 // Skip pushing arithmetic noops.
6789 if (!isIdentityFunction(llvm::dwarf::DW_OP_minus, Start)) {
6790 if (!pushSCEV(Start))
6791 return false;
6792 pushOperator(llvm::dwarf::DW_OP_minus);
6793 }
6794 if (!isIdentityFunction(llvm::dwarf::DW_OP_div, Stride)) {
6795 if (!pushSCEV(Stride))
6796 return false;
6797 pushOperator(llvm::dwarf::DW_OP_div);
6798 }
6799 return true;
6800 }
6801
6802 // Append the current expression and locations to a location list and an
6803 // expression list. Modify the DW_OP_LLVM_arg indexes to account for
6804 // the locations already present in the destination list.
6805 void appendToVectors(SmallVectorImpl<uint64_t> &DestExpr,
6806 SmallVectorImpl<Value *> &DestLocations) {
6807 assert(!DestLocations.empty() &&
6808 "Expected the locations vector to contain the IV");
6809 // The DWARF_OP_LLVM_arg arguments of the expression being appended must be
6810 // modified to account for the locations already in the destination vector.
6811 // All builders contain the IV as the first location op.
6812 assert(!LocationOps.empty() &&
6813 "Expected the location ops to contain the IV.");
6814 // DestIndexMap[n] contains the index in DestLocations for the nth
6815 // location in this SCEVDbgValueBuilder.
6816 SmallVector<uint64_t, 2> DestIndexMap;
6817 for (const auto &Op : LocationOps) {
6818 auto It = find(DestLocations, Op);
6819 if (It != DestLocations.end()) {
6820 // Location already exists in DestLocations, reuse existing ArgIndex.
6821 DestIndexMap.push_back(std::distance(DestLocations.begin(), It));
6822 continue;
6823 }
6824 // Location is not in DestLocations, add it.
6825 DestIndexMap.push_back(DestLocations.size());
6826 DestLocations.push_back(Op);
6827 }
6828
6829 for (const auto &Op : expr_ops()) {
6831 if (!Arg) {
6832 Op.appendToVector(DestExpr);
6833 continue;
6834 }
6835
6837 // `DW_OP_LLVM_arg n` represents the nth LocationOp in this SCEV,
6838 // DestIndexMap[n] contains its new index in DestLocations.
6839 uint64_t NewIndex = DestIndexMap[Arg.getIndex()];
6840 DestExpr.push_back(NewIndex);
6841 }
6842 }
6843};
6844
6845/// Holds all the required data to salvage a dbg.value using the pre-LSR SCEVs
6846/// and DIExpression.
6847struct DVIRecoveryRec {
6848 DVIRecoveryRec(DbgVariableRecord *DVR)
6849 : DbgRef(DVR), Expr(DVR->getExpression()), HadLocationArgList(false) {}
6850
6851 DbgVariableRecord *DbgRef;
6852 DIExpression *Expr;
6853 bool HadLocationArgList;
6854 SmallVector<WeakVH, 2> LocationOps;
6857
6858 void clear() {
6859 for (auto &RE : RecoveryExprs)
6860 RE.reset();
6861 RecoveryExprs.clear();
6862 }
6863
6864 ~DVIRecoveryRec() { clear(); }
6865};
6866} // namespace
6867
6868/// Returns the total number of DW_OP_llvm_arg operands in the expression.
6869/// This helps in determining if a DIArglist is necessary or can be omitted from
6870/// the dbg.value.
6872 auto expr_ops = ToDwarfOpIter(Expr);
6873 unsigned Count = 0;
6874 for (auto Op : expr_ops)
6875 if (Op.getOp() == dwarf::DW_OP_LLVM_arg)
6876 Count++;
6877 return Count;
6878}
6879
6880/// Overwrites DVI with the location and Ops as the DIExpression. This will
6881/// create an invalid expression if Ops has any dwarf::DW_OP_llvm_arg operands,
6882/// because a DIArglist is not created for the first argument of the dbg.value.
6883template <typename T>
6884static void updateDVIWithLocation(T &DbgVal, Value *Location,
6886 assert(numLLVMArgOps(Ops) == 0 && "Expected expression that does not "
6887 "contain any DW_OP_llvm_arg operands.");
6888 DbgVal.setRawLocation(ValueAsMetadata::get(Location));
6889 DbgVal.setExpression(DIExpression::get(DbgVal.getContext(), Ops));
6890}
6891
6892/// Overwrite DVI with locations placed into a DIArglist.
6893template <typename T>
6894static void updateDVIWithLocations(T &DbgVal,
6895 SmallVectorImpl<Value *> &Locations,
6897 assert(numLLVMArgOps(Ops) != 0 &&
6898 "Expected expression that references DIArglist locations using "
6899 "DW_OP_llvm_arg operands.");
6901 for (Value *V : Locations)
6902 MetadataLocs.push_back(ValueAsMetadata::get(V));
6903 auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
6904 DbgVal.setRawLocation(llvm::DIArgList::get(DbgVal.getContext(), ValArrayRef));
6905 DbgVal.setExpression(DIExpression::get(DbgVal.getContext(), Ops));
6906}
6907
6908/// Write the new expression and new location ops for the dbg.value. If possible
6909/// reduce the szie of the dbg.value by omitting DIArglist. This
6910/// can be omitted if:
6911/// 1. There is only a single location, refenced by a single DW_OP_llvm_arg.
6912/// 2. The DW_OP_LLVM_arg is the first operand in the expression.
6913static void UpdateDbgValue(DVIRecoveryRec &DVIRec,
6914 SmallVectorImpl<Value *> &NewLocationOps,
6916 DbgVariableRecord *DbgVal = DVIRec.DbgRef;
6917 unsigned NumLLVMArgs = numLLVMArgOps(NewExpr);
6918 if (NumLLVMArgs == 0) {
6919 // Location assumed to be on the stack.
6920 updateDVIWithLocation(*DbgVal, NewLocationOps[0], NewExpr);
6921 } else if (NumLLVMArgs == 1 && NewExpr[0] == dwarf::DW_OP_LLVM_arg) {
6922 // There is only a single DW_OP_llvm_arg at the start of the expression,
6923 // so it can be omitted along with DIArglist.
6924 assert(NewExpr[1] == 0 &&
6925 "Lone LLVM_arg in a DIExpression should refer to location-op 0.");
6927 updateDVIWithLocation(*DbgVal, NewLocationOps[0], ShortenedOps);
6928 } else {
6929 // Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
6930 updateDVIWithLocations(*DbgVal, NewLocationOps, NewExpr);
6931 }
6932
6933 // If the DIExpression was previously empty then add the stack terminator.
6934 // Non-empty expressions have only had elements inserted into them and so
6935 // the terminator should already be present e.g. stack_value or fragment.
6936 DIExpression *SalvageExpr = DbgVal->getExpression();
6937 if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
6938 SalvageExpr = DIExpression::append(SalvageExpr, {dwarf::DW_OP_stack_value});
6939 DbgVal->setExpression(SalvageExpr);
6940 }
6941}
6942
6943/// Cached location ops may be erased during LSR, in which case a poison is
6944/// required when restoring from the cache. The type of that location is no
6945/// longer available, so just use int8. The poison will be replaced by one or
6946/// more locations later when a SCEVDbgValueBuilder selects alternative
6947/// locations to use for the salvage.
6949 return (VH) ? VH : PoisonValue::get(llvm::Type::getInt8Ty(C));
6950}
6951
6952/// Restore the DVI's pre-LSR arguments. Substitute undef for any erased values.
6953static void restorePreTransformState(DVIRecoveryRec &DVIRec) {
6954 DbgVariableRecord *DbgVal = DVIRec.DbgRef;
6955 LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
6956 << "scev-salvage: post-LSR: " << *DbgVal << '\n');
6957 assert(DVIRec.Expr && "Expected an expression");
6958 DbgVal->setExpression(DVIRec.Expr);
6959
6960 // Even a single location-op may be inside a DIArgList and referenced with
6961 // DW_OP_LLVM_arg, which is valid only with a DIArgList.
6962 if (!DVIRec.HadLocationArgList) {
6963 assert(DVIRec.LocationOps.size() == 1 &&
6964 "Unexpected number of location ops.");
6965 // LSR's unsuccessful salvage attempt may have added DIArgList, which in
6966 // this case was not present before, so force the location back to a
6967 // single uncontained Value.
6968 Value *CachedValue =
6969 getValueOrPoison(DVIRec.LocationOps[0], DbgVal->getContext());
6970 DbgVal->setRawLocation(ValueAsMetadata::get(CachedValue));
6971 } else {
6973 for (WeakVH VH : DVIRec.LocationOps) {
6974 Value *CachedValue = getValueOrPoison(VH, DbgVal->getContext());
6975 MetadataLocs.push_back(ValueAsMetadata::get(CachedValue));
6976 }
6977 auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
6978 DbgVal->setRawLocation(
6979 llvm::DIArgList::get(DbgVal->getContext(), ValArrayRef));
6980 }
6981 LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << '\n');
6982}
6983
6985 llvm::PHINode *LSRInductionVar, DVIRecoveryRec &DVIRec,
6986 const SCEV *SCEVInductionVar,
6987 SCEVDbgValueBuilder IterCountExpr) {
6988
6989 if (!DVIRec.DbgRef->isKillLocation())
6990 return false;
6991
6992 // LSR may have caused several changes to the dbg.value in the failed salvage
6993 // attempt. So restore the DIExpression, the location ops and also the
6994 // location ops format, which is always DIArglist for multiple ops, but only
6995 // sometimes for a single op.
6997
6998 // LocationOpIndexMap[i] will store the post-LSR location index of
6999 // the non-optimised out location at pre-LSR index i.
7000 SmallVector<int64_t, 2> LocationOpIndexMap;
7001 LocationOpIndexMap.assign(DVIRec.LocationOps.size(), -1);
7002 SmallVector<Value *, 2> NewLocationOps;
7003 NewLocationOps.push_back(LSRInductionVar);
7004
7005 for (unsigned i = 0; i < DVIRec.LocationOps.size(); i++) {
7006 WeakVH VH = DVIRec.LocationOps[i];
7007 // Place the locations not optimised out in the list first, avoiding
7008 // inserts later. The map is used to update the DIExpression's
7009 // DW_OP_LLVM_arg arguments as the expression is updated.
7010 if (VH && !isa<UndefValue>(VH)) {
7011 NewLocationOps.push_back(VH);
7012 LocationOpIndexMap[i] = NewLocationOps.size() - 1;
7013 LLVM_DEBUG(dbgs() << "scev-salvage: Location index " << i
7014 << " now at index " << LocationOpIndexMap[i] << "\n");
7015 continue;
7016 }
7017
7018 // It's possible that a value referred to in the SCEV may have been
7019 // optimised out by LSR.
7020 if (SE.containsErasedValue(DVIRec.SCEVs[i]) ||
7021 SE.containsUndefs(DVIRec.SCEVs[i])) {
7022 LLVM_DEBUG(dbgs() << "scev-salvage: SCEV for location at index: " << i
7023 << " refers to a location that is now undef or erased. "
7024 "Salvage abandoned.\n");
7025 return false;
7026 }
7027
7028 LLVM_DEBUG(dbgs() << "scev-salvage: salvaging location at index " << i
7029 << " with SCEV: " << *DVIRec.SCEVs[i] << "\n");
7030
7031 DVIRec.RecoveryExprs[i] = std::make_unique<SCEVDbgValueBuilder>();
7032 SCEVDbgValueBuilder *SalvageExpr = DVIRec.RecoveryExprs[i].get();
7033
7034 // Create an offset-based salvage expression if possible, as it requires
7035 // less DWARF ops than an iteration count-based expression.
7036 if (std::optional<APInt> Offset =
7037 SE.computeConstantDifference(DVIRec.SCEVs[i], SCEVInductionVar)) {
7038 if (Offset->getSignificantBits() <= 64)
7039 SalvageExpr->createOffsetExpr(Offset->getSExtValue(), LSRInductionVar);
7040 else
7041 return false;
7042 } else if (!SalvageExpr->createIterCountExpr(DVIRec.SCEVs[i], IterCountExpr,
7043 SE))
7044 return false;
7045 }
7046
7047 // Merge the DbgValueBuilder generated expressions and the original
7048 // DIExpression, place the result into an new vector.
7050 if (DVIRec.Expr->getNumElements() == 0) {
7051 assert(DVIRec.RecoveryExprs.size() == 1 &&
7052 "Expected only a single recovery expression for an empty "
7053 "DIExpression.");
7054 assert(DVIRec.RecoveryExprs[0] &&
7055 "Expected a SCEVDbgSalvageBuilder for location 0");
7056 SCEVDbgValueBuilder *B = DVIRec.RecoveryExprs[0].get();
7057 B->appendToVectors(NewExpr, NewLocationOps);
7058 }
7059 for (const auto &Op : DVIRec.Expr->expr_ops()) {
7060 // Most Ops needn't be updated.
7062 if (!Arg) {
7063 Op.appendToVector(NewExpr);
7064 continue;
7065 }
7066
7067 uint64_t LocationArgIndex = Arg.getIndex();
7068 SCEVDbgValueBuilder *DbgBuilder =
7069 DVIRec.RecoveryExprs[LocationArgIndex].get();
7070 // The location doesn't have s SCEVDbgValueBuilder, so LSR did not
7071 // optimise it away. So just translate the argument to the updated
7072 // location index.
7073 if (!DbgBuilder) {
7074 NewExpr.push_back(dwarf::DW_OP_LLVM_arg);
7075 assert(LocationOpIndexMap[LocationArgIndex] != -1 &&
7076 "Expected a positive index for the location-op position.");
7077 NewExpr.push_back(LocationOpIndexMap[LocationArgIndex]);
7078 continue;
7079 }
7080 // The location has a recovery expression.
7081 DbgBuilder->appendToVectors(NewExpr, NewLocationOps);
7082 }
7083
7084 UpdateDbgValue(DVIRec, NewLocationOps, NewExpr);
7085 LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: " << *DVIRec.DbgRef << "\n");
7086 return true;
7087}
7088
7089/// Obtain an expression for the iteration count, then attempt to salvage the
7090/// dbg.value intrinsics.
7092 llvm::Loop *L, ScalarEvolution &SE, llvm::PHINode *LSRInductionVar,
7093 SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &DVIToUpdate) {
7094 if (DVIToUpdate.empty())
7095 return;
7096
7097 const llvm::SCEV *SCEVInductionVar = SE.getSCEV(LSRInductionVar);
7098 assert(SCEVInductionVar &&
7099 "Anticipated a SCEV for the post-LSR induction variable");
7100
7101 if (const SCEVAddRecExpr *IVAddRec =
7102 dyn_cast<SCEVAddRecExpr>(SCEVInductionVar)) {
7103 if (!IVAddRec->isAffine())
7104 return;
7105
7106 // Prevent translation using excessive resources.
7107 if (IVAddRec->getExpressionSize() > MaxSCEVSalvageExpressionSize)
7108 return;
7109
7110 // The iteration count is required to recover location values.
7111 SCEVDbgValueBuilder IterCountExpr;
7112 IterCountExpr.pushLocation(LSRInductionVar);
7113 if (!IterCountExpr.SCEVToIterCountExpr(*IVAddRec, SE))
7114 return;
7115
7116 LLVM_DEBUG(dbgs() << "scev-salvage: IV SCEV: " << *SCEVInductionVar
7117 << '\n');
7118
7119 for (auto &DVIRec : DVIToUpdate) {
7120 SalvageDVI(L, SE, LSRInductionVar, *DVIRec, SCEVInductionVar,
7121 IterCountExpr);
7122 }
7123 }
7124}
7125
7126/// Identify and cache salvageable DVI locations and expressions along with the
7127/// corresponding SCEV(s). Also ensure that the DVI is not deleted between
7128/// cacheing and salvaging.
7130 Loop *L, ScalarEvolution &SE,
7131 SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &SalvageableDVISCEVs) {
7132 for (const auto &B : L->getBlocks()) {
7133 for (auto &I : *B) {
7134 for (DbgVariableRecord &DbgVal : filterDbgVars(I.getDbgRecordRange())) {
7135 if (!DbgVal.isDbgValue() && !DbgVal.isDbgAssign())
7136 continue;
7137
7138 // Ensure that if any location op is undef that the dbg.vlue is not
7139 // cached.
7140 if (DbgVal.isKillLocation())
7141 continue;
7142
7143 // Check that the location op SCEVs are suitable for translation to
7144 // DIExpression.
7145 const auto &HasTranslatableLocationOps =
7146 [&](const DbgVariableRecord &DbgValToTranslate) -> bool {
7147 for (const auto LocOp : DbgValToTranslate.location_ops()) {
7148 if (!LocOp)
7149 return false;
7150
7151 if (!SE.isSCEVable(LocOp->getType()))
7152 return false;
7153
7154 const SCEV *S = SE.getSCEV(LocOp);
7155 if (SE.containsUndefs(S))
7156 return false;
7157 }
7158 return true;
7159 };
7160
7161 if (!HasTranslatableLocationOps(DbgVal))
7162 continue;
7163
7164 std::unique_ptr<DVIRecoveryRec> NewRec =
7165 std::make_unique<DVIRecoveryRec>(&DbgVal);
7166 // Each location Op may need a SCEVDbgValueBuilder in order to recover
7167 // it. Pre-allocating a vector will enable quick lookups of the builder
7168 // later during the salvage.
7169 NewRec->RecoveryExprs.resize(DbgVal.getNumVariableLocationOps());
7170 for (const auto LocOp : DbgVal.location_ops()) {
7171 NewRec->SCEVs.push_back(SE.getSCEV(LocOp));
7172 NewRec->LocationOps.push_back(LocOp);
7173 NewRec->HadLocationArgList = DbgVal.hasArgList();
7174 }
7175 SalvageableDVISCEVs.push_back(std::move(NewRec));
7176 }
7177 }
7178 }
7179}
7180
7181/// Ideally pick the PHI IV inserted by ScalarEvolutionExpander. As a fallback
7182/// any PHi from the loop header is usable, but may have less chance of
7183/// surviving subsequent transforms.
7185 const LSRInstance &LSR) {
7186
7187 auto IsSuitableIV = [&](PHINode *P) {
7188 if (!SE.isSCEVable(P->getType()))
7189 return false;
7190 if (const SCEVAddRecExpr *Rec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(P)))
7191 return Rec->isAffine() && !SE.containsUndefs(SE.getSCEV(P));
7192 return false;
7193 };
7194
7195 // For now, just pick the first IV that was generated and inserted by
7196 // ScalarEvolution. Ideally pick an IV that is unlikely to be optimised away
7197 // by subsequent transforms.
7198 for (const WeakVH &IV : LSR.getScalarEvolutionIVs()) {
7199 if (!IV)
7200 continue;
7201
7202 // There should only be PHI node IVs.
7203 PHINode *P = cast<PHINode>(&*IV);
7204
7205 if (IsSuitableIV(P))
7206 return P;
7207 }
7208
7209 for (PHINode &P : L.getHeader()->phis()) {
7210 if (IsSuitableIV(&P))
7211 return &P;
7212 }
7213 return nullptr;
7214}
7215
7217 DominatorTree &DT, LoopInfo &LI,
7218 const TargetTransformInfo &TTI,
7220 MemorySSA *MSSA, bool PreserveLCSSA) {
7221 const ScalarOptions &Opts = getScalarOptions(*L);
7222
7223 // Debug preservation - before we start removing anything identify which DVI
7224 // meet the salvageable criteria and store their DIExpression and SCEVs.
7225 SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> SalvageableDVIRecords;
7226 DbgGatherSalvagableDVI(L, SE, SalvageableDVIRecords);
7227
7228 bool Changed = false;
7229 std::unique_ptr<MemorySSAUpdater> MSSAU;
7230 if (MSSA)
7231 MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
7232
7233 // Run the main LSR transformation.
7234 const LSRInstance &Reducer = LSRInstance(Opts, L, IU, SE, DT, LI, TTI, AC,
7235 TLI, MSSAU.get(), PreserveLCSSA);
7236 Changed |= Reducer.getChanged();
7237
7238 // Remove any extra phis created by processing inner loops.
7239 Changed |= DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get());
7240 if (Opts.enable_lsr_phielim && L->isLoopSimplifyForm()) {
7242 SCEVExpander Rewriter(SE, "lsr", false);
7243#if LLVM_ENABLE_ABI_BREAKING_CHECKS
7244 Rewriter.setDebugType(DEBUG_TYPE);
7245#endif
7246 unsigned numFolded = Rewriter.replaceCongruentIVs(L, &DT, DeadInsts, &TTI);
7247 Rewriter.clear();
7248 if (numFolded) {
7249 Changed = true;
7251 MSSAU.get());
7252 DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get());
7253 }
7254 }
7255
7256 // We might end up with duplicate phi nodes when preserving LCSSA. Clean them
7257 // up here, as otherwise rewriteLoopExitValues will bail due to the extra
7258 // uses.
7260 L->getUniqueExitBlocks(ExitBlocks);
7261 for (BasicBlock *ExitBB : ExitBlocks)
7263
7264 // LSR may at times remove all uses of an induction variable from a loop.
7265 // The only remaining use is the PHI in the exit block.
7266 // When this is the case, if the exit value of the IV can be calculated using
7267 // SCEV, we can replace the exit block PHI with the final value of the IV and
7268 // skip the updates in each loop iteration.
7269 if (L->isRecursivelyLCSSAForm(DT, LI) && L->getExitBlock()) {
7271 SCEVExpander Rewriter(SE, "lsr", true);
7272 int Rewrites = rewriteLoopExitValues(L, &LI, &TLI, &SE, &TTI, Rewriter, &DT,
7273 UnusedIndVarInLoop, DeadInsts);
7274 Rewriter.clear();
7275 if (Rewrites) {
7276 Changed = true;
7278 MSSAU.get());
7279 DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get());
7280 }
7281 }
7282
7283 if (SalvageableDVIRecords.empty())
7284 return Changed;
7285
7286 // Obtain relevant IVs and attempt to rewrite the salvageable DVIs with
7287 // expressions composed using the derived iteration count.
7288 // TODO: Allow for multiple IV references for nested AddRecSCEVs
7289 for (const auto &L : LI) {
7290 if (llvm::PHINode *IV = GetInductionVariable(*L, SE, Reducer))
7291 DbgRewriteSalvageableDVIs(L, SE, IV, SalvageableDVIRecords);
7292 else {
7293 LLVM_DEBUG(dbgs() << "scev-salvage: SCEV salvaging not possible. An IV "
7294 "could not be identified.\n");
7295 }
7296 }
7297
7298 for (auto &Rec : SalvageableDVIRecords)
7299 Rec->clear();
7300 SalvageableDVIRecords.clear();
7301 return Changed;
7302}
7303
7304bool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager & /*LPM*/) {
7305 if (skipLoop(L))
7306 return false;
7307
7308 auto &IU = getAnalysis<IVUsersWrapperPass>().getIU();
7309 auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
7310 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
7311 auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
7312 const auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
7313 *L->getHeader()->getParent());
7314 auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
7315 *L->getHeader()->getParent());
7316 auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
7317 *L->getHeader()->getParent());
7318 auto *MSSAAnalysis = getAnalysisIfAvailable<MemorySSAWrapperPass>();
7319 MemorySSA *MSSA = nullptr;
7320 if (MSSAAnalysis)
7321 MSSA = &MSSAAnalysis->getMSSA();
7322 return ReduceLoopStrength(L, IU, SE, DT, LI, TTI, AC, TLI, MSSA,
7323 /*PreserveLCSSA=*/false);
7324}
7325
7328 LPMUpdater &) {
7329 if (!ReduceLoopStrength(&L, AM.getResult<IVUsersAnalysis>(L, AR), AR.SE,
7330 AR.DT, AR.LI, AR.TTI, AR.AC, AR.TLI, AR.MSSA,
7331 /*PreserveLCSSA=*/true))
7332 return PreservedAnalyses::all();
7333
7334 auto PA = getLoopPassPreservedAnalyses();
7335 if (AR.MSSA)
7336 PA.preserve<MemorySSAAnalysis>();
7337 return PA;
7338}
7339
7340char LoopStrengthReduce::ID = 0;
7341
7342INITIALIZE_PASS_BEGIN(LoopStrengthReduce, "loop-reduce",
7343 "Loop Strength Reduction", false, false)
7349INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
7350INITIALIZE_PASS_END(LoopStrengthReduce, "loop-reduce",
7351 "Loop Strength Reduction", false, false)
7352
7353Pass *llvm::createLoopStrengthReducePass() { return new LoopStrengthReduce(); }
#define Success
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
const TargetLowering & TLI
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AArch64 Predicate As Counter Loop Rewrites
const AArch64InstrInfo MachineBasicBlock::iterator InsertPt
uint64_t uint64_t
uint64_t Imm
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis false
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static bool isAddressUse(const Use &U)
The use is the address of the function as a value.
static const Function * getParent(const Value *V)
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition Compiler.h:702
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool isCanonical(const MDString *S)
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
This file contains constants used for implementing Dwarf debug support.
early cse Early CSE w MemorySSA
#define DEBUG_TYPE
Hexagon Hardware Loops
Module.h This file contains the declarations for the Module class.
This defines the Use class.
iv Induction Variable Users
Definition IVUsers.cpp:48
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition Lint.cpp:550
This header provides classes for managing per-loop analyses.
static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE, llvm::PHINode *LSRInductionVar, DVIRecoveryRec &DVIRec, const SCEV *SCEVInductionVar, SCEVDbgValueBuilder IterCountExpr)
static Value * getWideOperand(Value *Oper)
IVChain logic must consistently peek base TruncInst operands, so wrap it in a convenient helper.
static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE)
Return true if the given add can be sign-extended without changing its value.
static bool stressIVChain(const ScalarOptions &Opts)
static bool mayUsePostIncMode(const TargetTransformInfo &TTI, LSRUse &LU, const SCEV *S, const Loop *L, ScalarEvolution &SE)
Return true if the SCEV represents a value that may end up as a post-increment operation.
static void restorePreTransformState(DVIRecoveryRec &DVIRec)
Restore the DVI's pre-LSR arguments. Substitute undef for any erased values.
static bool containsAddRecDependentOnLoop(const SCEV *S, const Loop &L)
static User::op_iterator findIVOperand(User::op_iterator OI, User::op_iterator OE, Loop *L, ScalarEvolution &SE)
Helper for CollectChains that finds an IV operand (computed by an AddRec in this loop) within [OI,...
static bool isLegalUse(const TargetTransformInfo &TTI, Immediate MinOffset, Immediate MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, Immediate BaseOffset, bool HasBaseReg, int64_t Scale)
Test whether we know how to expand the current formula.
static void DbgGatherSalvagableDVI(Loop *L, ScalarEvolution &SE, SmallVector< std::unique_ptr< DVIRecoveryRec >, 2 > &SalvageableDVISCEVs)
Identify and cache salvageable DVI locations and expressions along with the corresponding SCEV(s).
static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE)
Return true if the given mul can be sign-extended without changing its value.
static const unsigned MaxSCEVSalvageExpressionSize
Limit the size of expression that SCEV-based salvaging will attempt to translate into a DIExpression.
static bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE)
Return true if this AddRec is already a phi in its loop.
static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI, const LSRUse &LU, const Formula &F, const Loop &L)
static void DoInitialMatch(const SCEV *S, Loop *L, SmallVectorImpl< SCEVUse > &Good, SmallVectorImpl< SCEVUse > &Bad, ScalarEvolution &SE)
Recursion helper for initialMatch.
static void updateDVIWithLocation(T &DbgVal, Value *Location, SmallVectorImpl< uint64_t > &Ops)
Overwrites DVI with the location and Ops as the DIExpression.
static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT, LoopInfo &LI, const TargetTransformInfo &TTI, AssumptionCache &AC, TargetLibraryInfo &TLI, MemorySSA *MSSA, bool PreserveLCSSA)
static bool isLegalAddImmediate(const TargetTransformInfo &TTI, Immediate Offset)
static Instruction * getFixupInsertPos(const TargetTransformInfo &TTI, const LSRFixup &Fixup, const LSRUse &LU, Instruction *IVIncInsertPos, DominatorTree &DT)
static const SCEV * getExprBase(const SCEV *S)
Return an approximation of this SCEV expression's "base", or NULL for any constant.
static llvm::PHINode * GetInductionVariable(const Loop &L, ScalarEvolution &SE, const LSRInstance &LSR)
Ideally pick the PHI IV inserted by ScalarEvolutionExpander.
static bool IsSimplerBaseSCEVForTarget(const TargetTransformInfo &TTI, ScalarEvolution &SE, const SCEV *Best, const SCEV *Reg, MemAccessTy AccessType)
static const unsigned MaxIVUsers
MaxIVUsers is an arbitrary threshold that provides an early opportunity for bail out.
static Immediate extractImmediate(const ScalarOptions &Opts, SCEVUse &S, ScalarEvolution &SE, bool PreferScalable=false)
If S involves the addition of a constant integer value, return that integer value,...
static bool isHighCostExpansion(const SCEV *S, SmallPtrSetImpl< const SCEV * > &Processed, ScalarEvolution &SE)
Check if expanding this expression is likely to incur significant cost.
static Value * getValueOrPoison(WeakVH &VH, LLVMContext &C)
Cached location ops may be erased during LSR, in which case a poison is required when restoring from ...
static MemAccessTy getAccessType(const TargetTransformInfo &TTI, Instruction *Inst, Value *OperandVal)
Return the type of the memory being accessed.
static unsigned numLLVMArgOps(SmallVectorImpl< uint64_t > &Expr)
Returns the total number of DW_OP_llvm_arg operands in the expression.
static bool isAlwaysFoldable(const ScalarOptions &Opts, const TargetTransformInfo &TTI, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, Immediate BaseOffset, bool HasBaseReg)
static void DbgRewriteSalvageableDVIs(llvm::Loop *L, ScalarEvolution &SE, llvm::PHINode *LSRInductionVar, SmallVector< std::unique_ptr< DVIRecoveryRec >, 2 > &DVIToUpdate)
Obtain an expression for the iteration count, then attempt to salvage the dbg.value intrinsics.
static void UpdateDbgValue(DVIRecoveryRec &DVIRec, SmallVectorImpl< Value * > &NewLocationOps, SmallVectorImpl< uint64_t > &NewExpr)
Write the new expression and new location ops for the dbg.value.
static bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE)
Return true if the given addrec can be sign-extended without changing its value.
static bool isAMCompletelyFolded(const TargetTransformInfo &TTI, const LSRUse &LU, const Formula &F)
Check if the addressing mode defined by F is completely folded in LU at isel time.
static Immediate extractImmediateOperand(const ScalarOptions &Opts, MutableArrayRef< SCEVUse > Ops, ScalarEvolution &SE, bool PreferScalable)
Extracts an immediate operand from Ops and replaces the operand with zero.
static void updateDVIWithLocations(T &DbgVal, SmallVectorImpl< Value * > &Locations, SmallVectorImpl< uint64_t > &Ops)
Overwrite DVI with locations placed into a DIArglist.
static bool canFoldIVIncExpr(const ScalarOptions &Opts, const SCEV *IncExpr, Instruction *UserInst, Value *Operand, const TargetTransformInfo &TTI)
Return true if the IVInc can be folded into an addressing mode.
static GlobalValue * ExtractSymbol(SCEVUse &S, ScalarEvolution &SE)
If S involves the addition of a GlobalValue address, return that symbol, and mutate S to point to a n...
static bool isProfitableChain(IVChain &Chain, SmallPtrSetImpl< Instruction * > &Users, ScalarEvolution &SE, const TargetTransformInfo &TTI)
Return true if the number of registers needed for the chain is estimated to be less than the number r...
static const SCEV * CollectSubexprs(const SCEV *S, const SCEVConstant *C, SmallVectorImpl< const SCEV * > &Ops, const Loop *L, ScalarEvolution &SE, unsigned Depth=0)
Split S into subexpressions which can be pulled out into separate registers.
static const SCEV * getExactSDiv(const SCEV *LHS, const SCEV *RHS, ScalarEvolution &SE, bool IgnoreSignificantBits=false)
Return an expression for LHS /s RHS, if it can be determined and if the remainder is known to be zero...
static const SCEV * getAnyExtendConsideringPostIncUses(ArrayRef< PostIncLoopSet > Loops, const SCEV *Expr, Type *ToTy, ScalarEvolution &SE)
Extend/Truncate Expr to ToTy considering post-inc uses in Loops.
static unsigned getSetupCost(const SCEV *Reg, unsigned Depth, const TargetTransformInfo &TTI)
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define G(x, y, z)
Definition MD5.cpp:55
Register Reg
This file exposes an interface to building/using memory SSA to walk memory instructions using a use/d...
#define T
uint64_t IntrinsicInst * II
#define P(N)
if(PassOpts->AAPipeline)
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
This file defines the PointerIntPair class.
SmallVector< MachineOperand, 4 > Cond
Remove Loads Into Fake Uses
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
SI optimize exec mask operations pre RA
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define LLVM_DEBUG(...)
Definition Debug.h:119
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
This pass exposes codegen information to IR-level passes.
virt reg Virtual Register Rewriter
Value * RHS
Value * LHS
BinaryOperator * Mul
static const uint32_t IV[8]
Definition blake3_impl.h:83
Class for arbitrary precision integers.
Definition APInt.h:78
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
bool isNegative() const
Determine sign of this APInt.
Definition APInt.h:325
LLVM_ABI APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
Definition APInt.cpp:1673
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1659
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
Definition APInt.h:1551
LLVM_ABI APInt srem(const APInt &RHS) const
Function for signed remainder operation.
Definition APInt.cpp:1774
bool isIntN(unsigned N) const
Check if this APInt has an N-bits unsigned integer value.
Definition APInt.h:428
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
LLVM_ABI AnalysisUsage & addRequiredID(const void *ID)
Definition Pass.cpp:294
AnalysisUsage & addPreservedID(const void *ID)
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition BasicBlock.h:509
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
void moveBefore(BasicBlock *MovePos)
Unlink this basic block from its current function and insert it into the function that MovePos lives ...
Definition BasicBlock.h:367
LLVM_ABI bool isLandingPad() const
Return true if this basic block is a landing pad.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
BinaryOps getOpcode() const
Definition InstrTypes.h:409
static LLVM_ABI BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
static LLVM_ABI Instruction::CastOps getCastOpcode(const Value *Val, bool SrcIsSigned, Type *Ty, bool DstIsSigned)
Returns the opcode necessary to cast Val into Ty using usual casting rules.
static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ ICMP_NE
not equal
Definition InstrTypes.h:762
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:852
Value * getCondition() const
static LLVM_ABI bool isValueValidForType(Type *Ty, uint64_t V)
This static method returns true if the type Ty is big enough to represent the value V.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
Definition Constants.h:135
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition Constants.h:174
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static LLVM_ABI DIArgList * get(LLVMContext &Context, ArrayRef< ValueAsMetadata * > Args)
DWARF expression.
iterator_range< expr_op_iterator > expr_ops() const
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
unsigned getNumElements() const
static LLVM_ABI void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
LLVM_ABI bool isComplex() const
Return whether the location is computed on the expression stack, meaning it cannot be a simple regist...
LLVM_ABI LLVMContext & getContext()
Record of a variable value-assignment, aka a non instruction representation of the dbg....
LLVM_ABI bool isKillLocation() const
void setRawLocation(Metadata *NewLocation)
Use of this should generally be avoided; instead, replaceVariableLocationOp and addVariableLocationOp...
void setExpression(DIExpression *NewExpr)
DIExpression * getExpression() const
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:856
NodeT * getBlock() const
DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
bool properlyDominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
properlyDominates - Returns true iff A dominates B and A != B.
Legacy analysis pass which computes a DominatorTree.
Definition Dominators.h:277
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
LLVM_ABI Instruction * findNearestCommonDominator(Instruction *I1, Instruction *I2) const
Find the nearest instruction I that dominates both I1 and I2, in the sense that a result produced bef...
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
PointerType * getType() const
Global values are always pointers.
IVStrideUse - Keep track of one use of a strided induction variable.
Definition IVUsers.h:36
void transformToPostInc(const Loop *L)
transformToPostInc - Transform the expression to post-inc form for the given loop.
Definition IVUsers.cpp:365
Value * getOperandValToReplace() const
getOperandValToReplace - Return the Value of the operand in the user instruction that this IVStrideUs...
Definition IVUsers.h:55
void setUser(Instruction *NewUser)
setUser - Assign a new user instruction for this use.
Definition IVUsers.h:49
Analysis pass that exposes the IVUsers for a loop.
Definition IVUsers.h:187
ilist< IVStrideUse >::const_iterator const_iterator
Definition IVUsers.h:143
iterator end()
Definition IVUsers.h:145
iterator begin()
Definition IVUsers.h:144
bool empty() const
Definition IVUsers.h:148
LLVM_ABI void print(raw_ostream &OS) const
CostType getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
LLVM_ABI bool isLifetimeStartOrEnd() const LLVM_READONLY
Return true if the instruction is a llvm.lifetime.start or llvm.lifetime.end marker.
LLVM_ABI unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI Type * getAccessType() const LLVM_READONLY
Return the type this instruction accesses in memory, if any.
iterator_range< user_iterator > users()
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
This class provides an interface for updating the loop pass manager based on mutations to the loop ne...
An instruction for reading from memory.
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getHeader() const
unsigned getLoopDepth() const
Return the nesting level of this loop.
The legacy pass manager's analysis pass to compute loop information.
Definition LoopInfo.h:623
LLVM_ABI PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U)
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1582
An analysis that produces MemorySSA for a function.
Definition MemorySSA.h:936
Encapsulates MemorySSA, including all data associated with memory accesses.
Definition MemorySSA.h:689
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:294
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
iterator_range< const_block_iterator > blocks() const
op_range incoming_values()
void setIncomingValue(unsigned i, Value *V)
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
static unsigned getIncomingValueNumForOperand(unsigned i)
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static LLVM_ABI PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Pass interface - Implemented by all 'passes'.
Definition Pass.h:97
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
This node represents an addition of some number of SCEVs.
This node represents a polynomial recurrence on the trip count of the specified loop.
LLVM_ABI const SCEV * evaluateAtIteration(const SCEV *It, ScalarEvolution &SE) const
Return the value of this chain of recurrences at the specified iteration number.
bool isAffine() const
Return true if this represents an expression A + B*x where A and B are loop invariant values.
SCEVUse getStepRecurrence(ScalarEvolution &SE) const
Constructs and returns the recurrence indicating how much this expression steps by.
This class represents a constant integer value.
ConstantInt * getValue() const
const APInt & getAPInt() const
This class uses information about analyze scalars to rewrite expressions in canonical form.
This node represents multiplication of some number of SCEVs.
ArrayRef< SCEVUse > operands() const
This means that we are dealing with an entirely unknown SCEV value, and only represent it as its LLVM...
This class represents an analyzed expression in the program.
unsigned short getExpressionSize() const
LLVM_ABI bool isZero() const
Return true if the expression is a constant zero.
LLVM_ABI ArrayRef< SCEVUse > operands() const
Return operands of this SCEV expression.
Type * getType() const
Return the LLVM type of this SCEV expression.
static constexpr auto FlagNone
SCEVTypes getSCEVType() const
The main scalar evolution driver.
LLVM_ABI const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
const SCEV * getZero(Type *Ty)
Return a SCEV for the constant 0 of a specific type.
LLVM_ABI const SCEV * getMinusSCEV(SCEVUse LHS, SCEVUse RHS, SCEVFlags Flags=SCEV::FlagNone, unsigned Depth=0)
Return LHS-RHS.
LLVM_ABI uint64_t getTypeSizeInBits(Type *Ty) const
Return the size in bits of the specified type, for which isSCEVable must return true.
LLVM_ABI const SCEV * getConstant(ConstantInt *V)
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
LLVM_ABI const SCEV * getNoopOrSignExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
LLVM_ABI SCEVUse getAddRecExpr(SCEVUse Start, SCEVUse Step, const Loop *L, SCEVFlagsPair Flags)
Get an add recurrence expression for the specified loop.
LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
LLVM_ABI SCEVUse getAddExpr(SmallVectorImpl< SCEVUse > &Ops, SCEVFlagsPair Flags={}, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
LLVM_ABI bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
LLVM_ABI Type * getEffectiveSCEVType(Type *Ty) const
Return a type with the same bitwidth as the given type and which represents how SCEV will treat the g...
LLVM_ABI const SCEV * getAnyExtendExpr(SCEVUse Op, Type *Ty)
getAnyExtendExpr - Return a SCEV for the given operand extended with unspecified bits out to the give...
LLVM_ABI const SCEV * getSignExtendExpr(SCEVUse Op, Type *Ty, unsigned Depth=0)
LLVM_ABI bool containsUndefs(const SCEV *S) const
Return true if the SCEV expression contains an undef value.
LLVM_ABI const SCEV * getExitCount(const Loop *L, const BasicBlock *ExitingBlock, ExitCountKind Kind=Exact)
Return the number of times the backedge executes before the given exit would be taken; if not exactly...
LLVM_ABI const SCEV * getVScale(Type *Ty)
LLVM_ABI SCEVUse getMulExpr(SmallVectorImpl< SCEVUse > &Ops, SCEVFlagsPair Flags={}, unsigned Depth=0)
Get a canonical multiply expression, or something simpler if possible.
LLVM_ABI bool hasComputableLoopEvolution(const SCEV *S, const Loop *L)
Return true if the given SCEV changes value in a known way in the specified loop.
LLVM_ABI const SCEV * getPointerBase(const SCEV *V)
Transitively follow the chain of pointer-type operands until reaching a SCEV that does not have a sin...
APInt getUnsignedRangeMax(const SCEV *S)
Determine the max of the unsigned range for a particular SCEV.
LLVM_ABI const SCEV * getUnknown(Value *V)
LLVM_ABI std::optional< APInt > computeConstantDifference(const SCEV *LHS, const SCEV *RHS)
Compute LHS - RHS and returns the result as an APInt if it is a constant, and std::nullopt if it isn'...
LLVM_ABI bool properlyDominates(const SCEV *S, const BasicBlock *BB)
Return true if elements that makes up the given SCEV properly dominate the specified basic block.
LLVM_ABI bool containsErasedValue(const SCEV *S) const
Return true if the SCEV expression contains a Value that has been optimised out and is now a nullptr.
LLVMContext & getContext() const
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:103
iterator end()
Get an iterator to the end of the SetVector.
Definition SetVector.h:118
iterator begin()
Get an iterator to the beginning of the SetVector.
Definition SetVector.h:112
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
int find_first() const
Returns the index of the first set bit, -1 if none of the bits are set.
SmallBitVector & set()
iterator_range< const_set_bits_iterator > set_bits() const
int find_next(unsigned Prev) const
Returns the index of the next set bit following the "Prev" bit.
size_type size() const
Returns the number of bits in this bitvector.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
size_type count() const
Returns the number of bits which are set.
SmallBitVector & reset()
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
void insert_range(Range &&R)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
typename SuperClass::const_iterator const_iterator
typename SuperClass::iterator iterator
void resize(size_type N)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
static StackOffset get(int64_t Fixed, int64_t Scalable)
Definition TypeSize.h:42
An instruction for storing to memory.
Provides information about what library functions are available for the current target.
Wrapper pass for TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
LLVM_ABI bool shouldDropLSRSolutionIfLessProfitable() const
Return true if LSR should drop a found solution if it's calculated to be less profitable than the bas...
LLVM_ABI bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1, const TargetTransformInfo::LSRCost &C2) const
Return true if LSR cost of C1 is lower than C2.
LLVM_ABI bool isIndexedStoreLegal(enum MemIndexedMode Mode, Type *Ty) const
LLVM_ABI unsigned getRegisterClassForType(bool Vector, Type *Ty=nullptr) const
LLVM_ABI bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace=0, Instruction *I=nullptr, int64_t ScalableOffset=0) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
@ TCK_RecipThroughput
Reciprocal throughput.
LLVM_ABI bool isIndexedLoadLegal(enum MemIndexedMode Mode, Type *Ty) const
LLVM_ABI bool isTypeLegal(Type *Ty) const
Return true if this type is legal.
LLVM_ABI bool isLegalAddImmediate(int64_t Imm) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
LLVM_ABI bool canSaveCmp(Loop *L, CondBrInst **BI, ScalarEvolution *SE, LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, TargetLibraryInfo *LibInfo) const
Return true if the target can save a compare for loop count, for example hardware loop saves a compar...
LLVM_ABI unsigned getNumberOfRegisters(unsigned ClassID) const
LLVM_ABI bool isLegalAddScalableImmediate(int64_t Imm) const
Return true if adding the specified scalable immediate is legal, that is the target has add instructi...
@ TCC_Free
Expected to fold away in lowering.
LLVM_ABI bool canMacroFuseCmp() const
Return true if the target can fuse a compare and branch.
AddressingModeKind
Which addressing mode Loop Strength Reduction will try to generate.
@ AMK_PostIndexed
Prefer post-indexed addressing mode.
@ AMK_PreIndexed
Prefer pre-indexed addressing mode.
@ AMK_None
Don't prefer any addressing mode.
LLVM_ABI bool preferLoopExitWithEqualityComparison(const ICmpInst &Cond, const APInt &PostIncExitValue) const
Return true if replacing the relational loop exit comparison Cond with an equality comparison against...
LLVM_ABI bool isTruncateFree(Type *Ty1, Type *Ty2) const
Return true if it's free to truncate a value of type Ty1 to type Ty2.
This class represents a truncation of integer types.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:292
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
LLVM_ABI bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
Definition Type.cpp:61
LLVM_ABI int getFPMantissaWidth() const
Return the width of the mantissa of this type.
Definition Type.cpp:222
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
Use * op_iterator
Definition User.h:254
op_range operands()
Definition User.h:267
op_iterator op_begin()
Definition User.h:259
void setOperand(unsigned i, Value *Val)
Definition User.h:212
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition User.cpp:25
Value * getOperand(unsigned i) const
Definition User.h:207
op_iterator op_end()
Definition User.h:261
static LLVM_ABI ValueAsMetadata * get(Value *V)
Definition Metadata.cpp:512
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
iterator_range< user_iterator > users()
Definition Value.h:428
LLVM_ABI void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
iterator_range< use_iterator > uses()
Definition Value.h:382
A nullable Value handle that is nullable.
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:210
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition DenseSet.h:188
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Entry
Definition COFF.h:862
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
bind_cst_ty m_scev_APInt(const APInt *&C)
Match an SCEV constant and bind it to an APInt.
match_bind< const SCEVMulExpr > m_scev_Mul(const SCEVMulExpr *&V)
bool match(const SCEV *S, const Pattern &P)
SCEVAffineAddRec_match< Op0_t, Op1_t, match_isa< const Loop > > m_scev_AffineAddRec(const Op0_t &Op0, const Op1_t &Op1)
cst_pred_ty< is_specific_cst > m_scev_SpecificInt(uint64_t V)
Match an SCEV constant with a plain unsigned integer.
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition Dwarf.h:149
@ DW_OP_LLVM_convert
Only used in LLVM metadata.
Definition Dwarf.h:145
constexpr double e
Sequence
A sequence of states that a pointer may go through in which an objc_retain and objc_release are actua...
Definition PtrState.h:41
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< PhiNode * > Phi
Definition RDFGraph.h:390
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
iterator end() const
Definition BasicBlock.h:89
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
LLVM_ABI iterator begin() const
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
unsigned KindType
For isa, dyn_cast, etc operations on TelemetryInfo.
Definition Telemetry.h:83
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Offset
Definition DWP.cpp:577
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1781
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
InstructionCost Cost
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition Utils.cpp:1837
@ Store
The extracted value is stored (ExtractElement only).
bool operator!=(uint64_t V1, const APInt &V2)
Definition APInt.h:2139
LLVM_ABI bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, SmallPtrSetImpl< PHINode * > *KnownNonDeadPHIs=nullptr)
Examine each PHI in the given block and delete it if it is dead.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2224
const ScalarOptions & getScalarOptions(const LLVMContext &C)
LLVM_ABI char & LoopSimplifyID
bool isa_and_nonnull(const Y &Val)
Definition Casting.h:676
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
DomTreeNodeBase< BasicBlock > DomTreeNode
Definition Dominators.h:65
AnalysisManager< Loop, LoopStandardAnalysisResults & > LoopAnalysisManager
The loop analysis manager.
LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, Value *&Start, Value *&Step)
Attempt to match a simple first order recurrence cycle of the form: iv = phi Ty [Start,...
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
LLVM_ABI void initializeLoopStrengthReducePass(PassRegistry &)
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
LLVM_ABI const SCEV * denormalizeForPostIncUse(const SCEV *S, const PostIncLoopSet &Loops, ScalarEvolution &SE)
Denormalize S to be post-increment for all loops present in Loops.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
LLVM_ABI Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI void SplitLandingPadPredecessors(BasicBlock *OrigBB, ArrayRef< BasicBlock * > Preds, const char *Suffix, const char *Suffix2, SmallVectorImpl< BasicBlock * > &NewBBs, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, bool PreserveLCSSA=false)
This method transforms the landing pad, OrigBB, by introducing two new basic blocks into the function...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI const SCEV * normalizeForPostIncUse(const SCEV *S, const PostIncLoopSet &Loops, ScalarEvolution &SE, bool CheckInvertible=true)
Normalize S to be post-increment for all loops present in Loops.
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
iterator_range(Container &&) -> iterator_range< llvm::detail::IterOfRange< Container > >
@ Other
Any other memory.
Definition ModRef.h:68
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
TargetTransformInfo TTI
@ Add
Sum of integers.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition STLExtras.h:2028
DWARFExpression::Operation Op
LLVM_ABI Pass * createLoopStrengthReducePass()
LLVM_ABI BasicBlock * SplitCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
If this edge is a critical edge, insert a new node to split the critical edge.
LLVM_ABI bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(SmallVectorImpl< WeakTrackingVH > &DeadInsts, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
Same functionality as RecursivelyDeleteTriviallyDeadInstructions, but allow instructions that are not...
Definition Local.cpp:537
constexpr unsigned BitWidth
LLVM_ABI bool formLCSSAForInstructions(SmallVectorImpl< Instruction * > &Worklist, const DominatorTree &DT, const LoopInfo &LI, ScalarEvolution *SE, SmallVectorImpl< PHINode * > *PHIsToRemove=nullptr, SmallVectorImpl< PHINode * > *InsertedPHIs=nullptr)
Ensures LCSSA form for every instruction from the Worklist in the scope of innermost containing loop.
Definition LCSSA.cpp:328
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI PreservedAnalyses getLoopPassPreservedAnalyses()
Returns the minimum set of Analyses that all loop passes must preserve.
SmallPtrSet< const Loop *, 2 > PostIncLoopSet
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
LLVM_ABI int rewriteLoopExitValues(Loop *L, LoopInfo *LI, TargetLibraryInfo *TLI, ScalarEvolution *SE, const TargetTransformInfo *TTI, SCEVExpander &Rewriter, DominatorTree *DT, ReplaceExitVal ReplaceExitValue, SmallVector< WeakTrackingVH, 16 > &DeadInsts)
If the final value of any expressions that are recurrent in the loop can be computed,...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
constexpr bool valueOr(BoolOrDefault X, bool Default)
@ UnusedIndVarInLoop
Definition LoopUtils.h:608
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
LLVM_ABI bool EliminateDuplicatePHINodes(BasicBlock *BB)
Check for and eliminate duplicate PHI nodes in this block.
Definition Local.cpp:1550
SCEVUseT< const SCEV * > SCEVUse
bool SCEVExprContains(const SCEV *Root, PredTy Pred)
Return true if any node in Root satisfies the predicate Pred.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
Attributes of a target dependent hardware loop.
The adaptor from a function pass to a loop pass computes these analyses and makes them available to t...
Information about a load/store intrinsic defined by the target.
Value * PtrVal
This is the pointer that the intrinsic is loading from or storing to.
SCEVPtrT getPointer() const