LLVM API Documentation

LoopStrengthReduce.cpp
Go to the documentation of this file.
00001 //===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This transformation analyzes and transforms the induction variables (and
00011 // computations derived from them) into forms suitable for efficient execution
00012 // on the target.
00013 //
00014 // This pass performs a strength reduction on array references inside loops that
00015 // have as one or more of their components the loop induction variable, it
00016 // rewrites expressions to take advantage of scaled-index addressing modes
00017 // available on the target, and it performs a variety of other optimizations
00018 // related to loop induction variables.
00019 //
00020 // Terminology note: this code has a lot of handling for "post-increment" or
00021 // "post-inc" users. This is not talking about post-increment addressing modes;
00022 // it is instead talking about code like this:
00023 //
00024 //   %i = phi [ 0, %entry ], [ %i.next, %latch ]
00025 //   ...
00026 //   %i.next = add %i, 1
00027 //   %c = icmp eq %i.next, %n
00028 //
00029 // The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however
00030 // it's useful to think about these as the same register, with some uses using
00031 // the value of the register before the add and some using // it after. In this
00032 // example, the icmp is a post-increment user, since it uses %i.next, which is
00033 // the value of the induction variable after the increment. The other common
00034 // case of post-increment users is users outside the loop.
00035 //
00036 // TODO: More sophistication in the way Formulae are generated and filtered.
00037 //
00038 // TODO: Handle multiple loops at a time.
00039 //
00040 // TODO: Should the addressing mode BaseGV be changed to a ConstantExpr instead
00041 //       of a GlobalValue?
00042 //
00043 // TODO: When truncation is free, truncate ICmp users' operands to make it a
00044 //       smaller encoding (on x86 at least).
00045 //
00046 // TODO: When a negated register is used by an add (such as in a list of
00047 //       multiple base registers, or as the increment expression in an addrec),
00048 //       we may not actually need both reg and (-1 * reg) in registers; the
00049 //       negation can be implemented by using a sub instead of an add. The
00050 //       lack of support for taking this into consideration when making
00051 //       register pressure decisions is partly worked around by the "Special"
00052 //       use kind.
00053 //
00054 //===----------------------------------------------------------------------===//
00055 
00056 #include "llvm/Transforms/Scalar.h"
00057 #include "llvm/ADT/DenseSet.h"
00058 #include "llvm/ADT/Hashing.h"
00059 #include "llvm/ADT/STLExtras.h"
00060 #include "llvm/ADT/SetVector.h"
00061 #include "llvm/ADT/SmallBitVector.h"
00062 #include "llvm/Analysis/IVUsers.h"
00063 #include "llvm/Analysis/LoopPass.h"
00064 #include "llvm/Analysis/ScalarEvolutionExpander.h"
00065 #include "llvm/Analysis/TargetTransformInfo.h"
00066 #include "llvm/IR/Constants.h"
00067 #include "llvm/IR/DerivedTypes.h"
00068 #include "llvm/IR/Dominators.h"
00069 #include "llvm/IR/Instructions.h"
00070 #include "llvm/IR/IntrinsicInst.h"
00071 #include "llvm/IR/ValueHandle.h"
00072 #include "llvm/Support/CommandLine.h"
00073 #include "llvm/Support/Debug.h"
00074 #include "llvm/Support/raw_ostream.h"
00075 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
00076 #include "llvm/Transforms/Utils/Local.h"
00077 #include <algorithm>
00078 using namespace llvm;
00079 
00080 #define DEBUG_TYPE "loop-reduce"
00081 
00082 /// MaxIVUsers is an arbitrary threshold that provides an early opportunitiy for
00083 /// bail out. This threshold is far beyond the number of users that LSR can
00084 /// conceivably solve, so it should not affect generated code, but catches the
00085 /// worst cases before LSR burns too much compile time and stack space.
00086 static const unsigned MaxIVUsers = 200;
00087 
00088 // Temporary flag to cleanup congruent phis after LSR phi expansion.
00089 // It's currently disabled until we can determine whether it's truly useful or
00090 // not. The flag should be removed after the v3.0 release.
00091 // This is now needed for ivchains.
00092 static cl::opt<bool> EnablePhiElim(
00093   "enable-lsr-phielim", cl::Hidden, cl::init(true),
00094   cl::desc("Enable LSR phi elimination"));
00095 
00096 #ifndef NDEBUG
00097 // Stress test IV chain generation.
00098 static cl::opt<bool> StressIVChain(
00099   "stress-ivchain", cl::Hidden, cl::init(false),
00100   cl::desc("Stress test LSR IV chains"));
00101 #else
00102 static bool StressIVChain = false;
00103 #endif
00104 
00105 namespace {
00106 
00107 /// RegSortData - This class holds data which is used to order reuse candidates.
00108 class RegSortData {
00109 public:
00110   /// UsedByIndices - This represents the set of LSRUse indices which reference
00111   /// a particular register.
00112   SmallBitVector UsedByIndices;
00113 
00114   RegSortData() {}
00115 
00116   void print(raw_ostream &OS) const;
00117   void dump() const;
00118 };
00119 
00120 }
00121 
00122 void RegSortData::print(raw_ostream &OS) const {
00123   OS << "[NumUses=" << UsedByIndices.count() << ']';
00124 }
00125 
00126 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
00127 void RegSortData::dump() const {
00128   print(errs()); errs() << '\n';
00129 }
00130 #endif
00131 
00132 namespace {
00133 
00134 /// RegUseTracker - Map register candidates to information about how they are
00135 /// used.
00136 class RegUseTracker {
00137   typedef DenseMap<const SCEV *, RegSortData> RegUsesTy;
00138 
00139   RegUsesTy RegUsesMap;
00140   SmallVector<const SCEV *, 16> RegSequence;
00141 
00142 public:
00143   void CountRegister(const SCEV *Reg, size_t LUIdx);
00144   void DropRegister(const SCEV *Reg, size_t LUIdx);
00145   void SwapAndDropUse(size_t LUIdx, size_t LastLUIdx);
00146 
00147   bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const;
00148 
00149   const SmallBitVector &getUsedByIndices(const SCEV *Reg) const;
00150 
00151   void clear();
00152 
00153   typedef SmallVectorImpl<const SCEV *>::iterator iterator;
00154   typedef SmallVectorImpl<const SCEV *>::const_iterator const_iterator;
00155   iterator begin() { return RegSequence.begin(); }
00156   iterator end()   { return RegSequence.end(); }
00157   const_iterator begin() const { return RegSequence.begin(); }
00158   const_iterator end() const   { return RegSequence.end(); }
00159 };
00160 
00161 }
00162 
00163 void
00164 RegUseTracker::CountRegister(const SCEV *Reg, size_t LUIdx) {
00165   std::pair<RegUsesTy::iterator, bool> Pair =
00166     RegUsesMap.insert(std::make_pair(Reg, RegSortData()));
00167   RegSortData &RSD = Pair.first->second;
00168   if (Pair.second)
00169     RegSequence.push_back(Reg);
00170   RSD.UsedByIndices.resize(std::max(RSD.UsedByIndices.size(), LUIdx + 1));
00171   RSD.UsedByIndices.set(LUIdx);
00172 }
00173 
00174 void
00175 RegUseTracker::DropRegister(const SCEV *Reg, size_t LUIdx) {
00176   RegUsesTy::iterator It = RegUsesMap.find(Reg);
00177   assert(It != RegUsesMap.end());
00178   RegSortData &RSD = It->second;
00179   assert(RSD.UsedByIndices.size() > LUIdx);
00180   RSD.UsedByIndices.reset(LUIdx);
00181 }
00182 
00183 void
00184 RegUseTracker::SwapAndDropUse(size_t LUIdx, size_t LastLUIdx) {
00185   assert(LUIdx <= LastLUIdx);
00186 
00187   // Update RegUses. The data structure is not optimized for this purpose;
00188   // we must iterate through it and update each of the bit vectors.
00189   for (RegUsesTy::iterator I = RegUsesMap.begin(), E = RegUsesMap.end();
00190        I != E; ++I) {
00191     SmallBitVector &UsedByIndices = I->second.UsedByIndices;
00192     if (LUIdx < UsedByIndices.size())
00193       UsedByIndices[LUIdx] =
00194         LastLUIdx < UsedByIndices.size() ? UsedByIndices[LastLUIdx] : 0;
00195     UsedByIndices.resize(std::min(UsedByIndices.size(), LastLUIdx));
00196   }
00197 }
00198 
00199 bool
00200 RegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const {
00201   RegUsesTy::const_iterator I = RegUsesMap.find(Reg);
00202   if (I == RegUsesMap.end())
00203     return false;
00204   const SmallBitVector &UsedByIndices = I->second.UsedByIndices;
00205   int i = UsedByIndices.find_first();
00206   if (i == -1) return false;
00207   if ((size_t)i != LUIdx) return true;
00208   return UsedByIndices.find_next(i) != -1;
00209 }
00210 
00211 const SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const {
00212   RegUsesTy::const_iterator I = RegUsesMap.find(Reg);
00213   assert(I != RegUsesMap.end() && "Unknown register!");
00214   return I->second.UsedByIndices;
00215 }
00216 
00217 void RegUseTracker::clear() {
00218   RegUsesMap.clear();
00219   RegSequence.clear();
00220 }
00221 
00222 namespace {
00223 
00224 /// Formula - This class holds information that describes a formula for
00225 /// computing satisfying a use. It may include broken-out immediates and scaled
00226 /// registers.
00227 struct Formula {
00228   /// Global base address used for complex addressing.
00229   GlobalValue *BaseGV;
00230 
00231   /// Base offset for complex addressing.
00232   int64_t BaseOffset;
00233 
00234   /// Whether any complex addressing has a base register.
00235   bool HasBaseReg;
00236 
00237   /// The scale of any complex addressing.
00238   int64_t Scale;
00239 
00240   /// BaseRegs - The list of "base" registers for this use. When this is
00241   /// non-empty. The canonical representation of a formula is
00242   /// 1. BaseRegs.size > 1 implies ScaledReg != NULL and
00243   /// 2. ScaledReg != NULL implies Scale != 1 || !BaseRegs.empty().
00244   /// #1 enforces that the scaled register is always used when at least two
00245   /// registers are needed by the formula: e.g., reg1 + reg2 is reg1 + 1 * reg2.
00246   /// #2 enforces that 1 * reg is reg.
00247   /// This invariant can be temporarly broken while building a formula.
00248   /// However, every formula inserted into the LSRInstance must be in canonical
00249   /// form.
00250   SmallVector<const SCEV *, 4> BaseRegs;
00251 
00252   /// ScaledReg - The 'scaled' register for this use. This should be non-null
00253   /// when Scale is not zero.
00254   const SCEV *ScaledReg;
00255 
00256   /// UnfoldedOffset - An additional constant offset which added near the
00257   /// use. This requires a temporary register, but the offset itself can
00258   /// live in an add immediate field rather than a register.
00259   int64_t UnfoldedOffset;
00260 
00261   Formula()
00262       : BaseGV(nullptr), BaseOffset(0), HasBaseReg(false), Scale(0),
00263         ScaledReg(nullptr), UnfoldedOffset(0) {}
00264 
00265   void InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE);
00266 
00267   bool isCanonical() const;
00268 
00269   void Canonicalize();
00270 
00271   bool Unscale();
00272 
00273   size_t getNumRegs() const;
00274   Type *getType() const;
00275 
00276   void DeleteBaseReg(const SCEV *&S);
00277 
00278   bool referencesReg(const SCEV *S) const;
00279   bool hasRegsUsedByUsesOtherThan(size_t LUIdx,
00280                                   const RegUseTracker &RegUses) const;
00281 
00282   void print(raw_ostream &OS) const;
00283   void dump() const;
00284 };
00285 
00286 }
00287 
00288 /// DoInitialMatch - Recursion helper for InitialMatch.
00289 static void DoInitialMatch(const SCEV *S, Loop *L,
00290                            SmallVectorImpl<const SCEV *> &Good,
00291                            SmallVectorImpl<const SCEV *> &Bad,
00292                            ScalarEvolution &SE) {
00293   // Collect expressions which properly dominate the loop header.
00294   if (SE.properlyDominates(S, L->getHeader())) {
00295     Good.push_back(S);
00296     return;
00297   }
00298 
00299   // Look at add operands.
00300   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
00301     for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
00302          I != E; ++I)
00303       DoInitialMatch(*I, L, Good, Bad, SE);
00304     return;
00305   }
00306 
00307   // Look at addrec operands.
00308   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S))
00309     if (!AR->getStart()->isZero()) {
00310       DoInitialMatch(AR->getStart(), L, Good, Bad, SE);
00311       DoInitialMatch(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0),
00312                                       AR->getStepRecurrence(SE),
00313                                       // FIXME: AR->getNoWrapFlags()
00314                                       AR->getLoop(), SCEV::FlagAnyWrap),
00315                      L, Good, Bad, SE);
00316       return;
00317     }
00318 
00319   // Handle a multiplication by -1 (negation) if it didn't fold.
00320   if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S))
00321     if (Mul->getOperand(0)->isAllOnesValue()) {
00322       SmallVector<const SCEV *, 4> Ops(Mul->op_begin()+1, Mul->op_end());
00323       const SCEV *NewMul = SE.getMulExpr(Ops);
00324 
00325       SmallVector<const SCEV *, 4> MyGood;
00326       SmallVector<const SCEV *, 4> MyBad;
00327       DoInitialMatch(NewMul, L, MyGood, MyBad, SE);
00328       const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue(
00329         SE.getEffectiveSCEVType(NewMul->getType())));
00330       for (SmallVectorImpl<const SCEV *>::const_iterator I = MyGood.begin(),
00331            E = MyGood.end(); I != E; ++I)
00332         Good.push_back(SE.getMulExpr(NegOne, *I));
00333       for (SmallVectorImpl<const SCEV *>::const_iterator I = MyBad.begin(),
00334            E = MyBad.end(); I != E; ++I)
00335         Bad.push_back(SE.getMulExpr(NegOne, *I));
00336       return;
00337     }
00338 
00339   // Ok, we can't do anything interesting. Just stuff the whole thing into a
00340   // register and hope for the best.
00341   Bad.push_back(S);
00342 }
00343 
00344 /// InitialMatch - Incorporate loop-variant parts of S into this Formula,
00345 /// attempting to keep all loop-invariant and loop-computable values in a
00346 /// single base register.
00347 void Formula::InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) {
00348   SmallVector<const SCEV *, 4> Good;
00349   SmallVector<const SCEV *, 4> Bad;
00350   DoInitialMatch(S, L, Good, Bad, SE);
00351   if (!Good.empty()) {
00352     const SCEV *Sum = SE.getAddExpr(Good);
00353     if (!Sum->isZero())
00354       BaseRegs.push_back(Sum);
00355     HasBaseReg = true;
00356   }
00357   if (!Bad.empty()) {
00358     const SCEV *Sum = SE.getAddExpr(Bad);
00359     if (!Sum->isZero())
00360       BaseRegs.push_back(Sum);
00361     HasBaseReg = true;
00362   }
00363   Canonicalize();
00364 }
00365 
00366 /// \brief Check whether or not this formula statisfies the canonical
00367 /// representation.
00368 /// \see Formula::BaseRegs.
00369 bool Formula::isCanonical() const {
00370   if (ScaledReg)
00371     return Scale != 1 || !BaseRegs.empty();
00372   return BaseRegs.size() <= 1;
00373 }
00374 
00375 /// \brief Helper method to morph a formula into its canonical representation.
00376 /// \see Formula::BaseRegs.
00377 /// Every formula having more than one base register, must use the ScaledReg
00378 /// field. Otherwise, we would have to do special cases everywhere in LSR
00379 /// to treat reg1 + reg2 + ... the same way as reg1 + 1*reg2 + ...
00380 /// On the other hand, 1*reg should be canonicalized into reg.
00381 void Formula::Canonicalize() {
00382   if (isCanonical())
00383     return;
00384   // So far we did not need this case. This is easy to implement but it is
00385   // useless to maintain dead code. Beside it could hurt compile time.
00386   assert(!BaseRegs.empty() && "1*reg => reg, should not be needed.");
00387   // Keep the invariant sum in BaseRegs and one of the variant sum in ScaledReg.
00388   ScaledReg = BaseRegs.back();
00389   BaseRegs.pop_back();
00390   Scale = 1;
00391   size_t BaseRegsSize = BaseRegs.size();
00392   size_t Try = 0;
00393   // If ScaledReg is an invariant, try to find a variant expression.
00394   while (Try < BaseRegsSize && !isa<SCEVAddRecExpr>(ScaledReg))
00395     std::swap(ScaledReg, BaseRegs[Try++]);
00396 }
00397 
00398 /// \brief Get rid of the scale in the formula.
00399 /// In other words, this method morphes reg1 + 1*reg2 into reg1 + reg2.
00400 /// \return true if it was possible to get rid of the scale, false otherwise.
00401 /// \note After this operation the formula may not be in the canonical form.
00402 bool Formula::Unscale() {
00403   if (Scale != 1)
00404     return false;
00405   Scale = 0;
00406   BaseRegs.push_back(ScaledReg);
00407   ScaledReg = nullptr;
00408   return true;
00409 }
00410 
00411 /// getNumRegs - Return the total number of register operands used by this
00412 /// formula. This does not include register uses implied by non-constant
00413 /// addrec strides.
00414 size_t Formula::getNumRegs() const {
00415   return !!ScaledReg + BaseRegs.size();
00416 }
00417 
00418 /// getType - Return the type of this formula, if it has one, or null
00419 /// otherwise. This type is meaningless except for the bit size.
00420 Type *Formula::getType() const {
00421   return !BaseRegs.empty() ? BaseRegs.front()->getType() :
00422          ScaledReg ? ScaledReg->getType() :
00423          BaseGV ? BaseGV->getType() :
00424          nullptr;
00425 }
00426 
00427 /// DeleteBaseReg - Delete the given base reg from the BaseRegs list.
00428 void Formula::DeleteBaseReg(const SCEV *&S) {
00429   if (&S != &BaseRegs.back())
00430     std::swap(S, BaseRegs.back());
00431   BaseRegs.pop_back();
00432 }
00433 
00434 /// referencesReg - Test if this formula references the given register.
00435 bool Formula::referencesReg(const SCEV *S) const {
00436   return S == ScaledReg ||
00437          std::find(BaseRegs.begin(), BaseRegs.end(), S) != BaseRegs.end();
00438 }
00439 
00440 /// hasRegsUsedByUsesOtherThan - Test whether this formula uses registers
00441 /// which are used by uses other than the use with the given index.
00442 bool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx,
00443                                          const RegUseTracker &RegUses) const {
00444   if (ScaledReg)
00445     if (RegUses.isRegUsedByUsesOtherThan(ScaledReg, LUIdx))
00446       return true;
00447   for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(),
00448        E = BaseRegs.end(); I != E; ++I)
00449     if (RegUses.isRegUsedByUsesOtherThan(*I, LUIdx))
00450       return true;
00451   return false;
00452 }
00453 
00454 void Formula::print(raw_ostream &OS) const {
00455   bool First = true;
00456   if (BaseGV) {
00457     if (!First) OS << " + "; else First = false;
00458     BaseGV->printAsOperand(OS, /*PrintType=*/false);
00459   }
00460   if (BaseOffset != 0) {
00461     if (!First) OS << " + "; else First = false;
00462     OS << BaseOffset;
00463   }
00464   for (SmallVectorImpl<const SCEV *>::const_iterator I = BaseRegs.begin(),
00465        E = BaseRegs.end(); I != E; ++I) {
00466     if (!First) OS << " + "; else First = false;
00467     OS << "reg(" << **I << ')';
00468   }
00469   if (HasBaseReg && BaseRegs.empty()) {
00470     if (!First) OS << " + "; else First = false;
00471     OS << "**error: HasBaseReg**";
00472   } else if (!HasBaseReg && !BaseRegs.empty()) {
00473     if (!First) OS << " + "; else First = false;
00474     OS << "**error: !HasBaseReg**";
00475   }
00476   if (Scale != 0) {
00477     if (!First) OS << " + "; else First = false;
00478     OS << Scale << "*reg(";
00479     if (ScaledReg)
00480       OS << *ScaledReg;
00481     else
00482       OS << "<unknown>";
00483     OS << ')';
00484   }
00485   if (UnfoldedOffset != 0) {
00486     if (!First) OS << " + ";
00487     OS << "imm(" << UnfoldedOffset << ')';
00488   }
00489 }
00490 
00491 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
00492 void Formula::dump() const {
00493   print(errs()); errs() << '\n';
00494 }
00495 #endif
00496 
00497 /// isAddRecSExtable - Return true if the given addrec can be sign-extended
00498 /// without changing its value.
00499 static bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
00500   Type *WideTy =
00501     IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(AR->getType()) + 1);
00502   return isa<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy));
00503 }
00504 
00505 /// isAddSExtable - Return true if the given add can be sign-extended
00506 /// without changing its value.
00507 static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) {
00508   Type *WideTy =
00509     IntegerType::get(SE.getContext(), SE.getTypeSizeInBits(A->getType()) + 1);
00510   return isa<SCEVAddExpr>(SE.getSignExtendExpr(A, WideTy));
00511 }
00512 
00513 /// isMulSExtable - Return true if the given mul can be sign-extended
00514 /// without changing its value.
00515 static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) {
00516   Type *WideTy =
00517     IntegerType::get(SE.getContext(),
00518                      SE.getTypeSizeInBits(M->getType()) * M->getNumOperands());
00519   return isa<SCEVMulExpr>(SE.getSignExtendExpr(M, WideTy));
00520 }
00521 
00522 /// getExactSDiv - Return an expression for LHS /s RHS, if it can be determined
00523 /// and if the remainder is known to be zero,  or null otherwise. If
00524 /// IgnoreSignificantBits is true, expressions like (X * Y) /s Y are simplified
00525 /// to Y, ignoring that the multiplication may overflow, which is useful when
00526 /// the result will be used in a context where the most significant bits are
00527 /// ignored.
00528 static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
00529                                 ScalarEvolution &SE,
00530                                 bool IgnoreSignificantBits = false) {
00531   // Handle the trivial case, which works for any SCEV type.
00532   if (LHS == RHS)
00533     return SE.getConstant(LHS->getType(), 1);
00534 
00535   // Handle a few RHS special cases.
00536   const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS);
00537   if (RC) {
00538     const APInt &RA = RC->getValue()->getValue();
00539     // Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do
00540     // some folding.
00541     if (RA.isAllOnesValue())
00542       return SE.getMulExpr(LHS, RC);
00543     // Handle x /s 1 as x.
00544     if (RA == 1)
00545       return LHS;
00546   }
00547 
00548   // Check for a division of a constant by a constant.
00549   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(LHS)) {
00550     if (!RC)
00551       return nullptr;
00552     const APInt &LA = C->getValue()->getValue();
00553     const APInt &RA = RC->getValue()->getValue();
00554     if (LA.srem(RA) != 0)
00555       return nullptr;
00556     return SE.getConstant(LA.sdiv(RA));
00557   }
00558 
00559   // Distribute the sdiv over addrec operands, if the addrec doesn't overflow.
00560   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS)) {
00561     if (IgnoreSignificantBits || isAddRecSExtable(AR, SE)) {
00562       const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE,
00563                                       IgnoreSignificantBits);
00564       if (!Step) return nullptr;
00565       const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE,
00566                                        IgnoreSignificantBits);
00567       if (!Start) return nullptr;
00568       // FlagNW is independent of the start value, step direction, and is
00569       // preserved with smaller magnitude steps.
00570       // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
00571       return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagAnyWrap);
00572     }
00573     return nullptr;
00574   }
00575 
00576   // Distribute the sdiv over add operands, if the add doesn't overflow.
00577   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(LHS)) {
00578     if (IgnoreSignificantBits || isAddSExtable(Add, SE)) {
00579       SmallVector<const SCEV *, 8> Ops;
00580       for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
00581            I != E; ++I) {
00582         const SCEV *Op = getExactSDiv(*I, RHS, SE,
00583                                       IgnoreSignificantBits);
00584         if (!Op) return nullptr;
00585         Ops.push_back(Op);
00586       }
00587       return SE.getAddExpr(Ops);
00588     }
00589     return nullptr;
00590   }
00591 
00592   // Check for a multiply operand that we can pull RHS out of.
00593   if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS)) {
00594     if (IgnoreSignificantBits || isMulSExtable(Mul, SE)) {
00595       SmallVector<const SCEV *, 4> Ops;
00596       bool Found = false;
00597       for (SCEVMulExpr::op_iterator I = Mul->op_begin(), E = Mul->op_end();
00598            I != E; ++I) {
00599         const SCEV *S = *I;
00600         if (!Found)
00601           if (const SCEV *Q = getExactSDiv(S, RHS, SE,
00602                                            IgnoreSignificantBits)) {
00603             S = Q;
00604             Found = true;
00605           }
00606         Ops.push_back(S);
00607       }
00608       return Found ? SE.getMulExpr(Ops) : nullptr;
00609     }
00610     return nullptr;
00611   }
00612 
00613   // Otherwise we don't know.
00614   return nullptr;
00615 }
00616 
00617 /// ExtractImmediate - If S involves the addition of a constant integer value,
00618 /// return that integer value, and mutate S to point to a new SCEV with that
00619 /// value excluded.
00620 static int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) {
00621   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) {
00622     if (C->getValue()->getValue().getMinSignedBits() <= 64) {
00623       S = SE.getConstant(C->getType(), 0);
00624       return C->getValue()->getSExtValue();
00625     }
00626   } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
00627     SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end());
00628     int64_t Result = ExtractImmediate(NewOps.front(), SE);
00629     if (Result != 0)
00630       S = SE.getAddExpr(NewOps);
00631     return Result;
00632   } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
00633     SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end());
00634     int64_t Result = ExtractImmediate(NewOps.front(), SE);
00635     if (Result != 0)
00636       S = SE.getAddRecExpr(NewOps, AR->getLoop(),
00637                            // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
00638                            SCEV::FlagAnyWrap);
00639     return Result;
00640   }
00641   return 0;
00642 }
00643 
00644 /// ExtractSymbol - If S involves the addition of a GlobalValue address,
00645 /// return that symbol, and mutate S to point to a new SCEV with that
00646 /// value excluded.
00647 static GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) {
00648   if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
00649     if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue())) {
00650       S = SE.getConstant(GV->getType(), 0);
00651       return GV;
00652     }
00653   } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
00654     SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end());
00655     GlobalValue *Result = ExtractSymbol(NewOps.back(), SE);
00656     if (Result)
00657       S = SE.getAddExpr(NewOps);
00658     return Result;
00659   } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
00660     SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end());
00661     GlobalValue *Result = ExtractSymbol(NewOps.front(), SE);
00662     if (Result)
00663       S = SE.getAddRecExpr(NewOps, AR->getLoop(),
00664                            // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
00665                            SCEV::FlagAnyWrap);
00666     return Result;
00667   }
00668   return nullptr;
00669 }
00670 
00671 /// isAddressUse - Returns true if the specified instruction is using the
00672 /// specified value as an address.
00673 static bool isAddressUse(Instruction *Inst, Value *OperandVal) {
00674   bool isAddress = isa<LoadInst>(Inst);
00675   if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
00676     if (SI->getOperand(1) == OperandVal)
00677       isAddress = true;
00678   } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
00679     // Addressing modes can also be folded into prefetches and a variety
00680     // of intrinsics.
00681     switch (II->getIntrinsicID()) {
00682       default: break;
00683       case Intrinsic::prefetch:
00684       case Intrinsic::x86_sse_storeu_ps:
00685       case Intrinsic::x86_sse2_storeu_pd:
00686       case Intrinsic::x86_sse2_storeu_dq:
00687       case Intrinsic::x86_sse2_storel_dq:
00688         if (II->getArgOperand(0) == OperandVal)
00689           isAddress = true;
00690         break;
00691     }
00692   }
00693   return isAddress;
00694 }
00695 
00696 /// getAccessType - Return the type of the memory being accessed.
00697 static Type *getAccessType(const Instruction *Inst) {
00698   Type *AccessTy = Inst->getType();
00699   if (const StoreInst *SI = dyn_cast<StoreInst>(Inst))
00700     AccessTy = SI->getOperand(0)->getType();
00701   else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
00702     // Addressing modes can also be folded into prefetches and a variety
00703     // of intrinsics.
00704     switch (II->getIntrinsicID()) {
00705     default: break;
00706     case Intrinsic::x86_sse_storeu_ps:
00707     case Intrinsic::x86_sse2_storeu_pd:
00708     case Intrinsic::x86_sse2_storeu_dq:
00709     case Intrinsic::x86_sse2_storel_dq:
00710       AccessTy = II->getArgOperand(0)->getType();
00711       break;
00712     }
00713   }
00714 
00715   // All pointers have the same requirements, so canonicalize them to an
00716   // arbitrary pointer type to minimize variation.
00717   if (PointerType *PTy = dyn_cast<PointerType>(AccessTy))
00718     AccessTy = PointerType::get(IntegerType::get(PTy->getContext(), 1),
00719                                 PTy->getAddressSpace());
00720 
00721   return AccessTy;
00722 }
00723 
00724 /// isExistingPhi - Return true if this AddRec is already a phi in its loop.
00725 static bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
00726   for (BasicBlock::iterator I = AR->getLoop()->getHeader()->begin();
00727        PHINode *PN = dyn_cast<PHINode>(I); ++I) {
00728     if (SE.isSCEVable(PN->getType()) &&
00729         (SE.getEffectiveSCEVType(PN->getType()) ==
00730          SE.getEffectiveSCEVType(AR->getType())) &&
00731         SE.getSCEV(PN) == AR)
00732       return true;
00733   }
00734   return false;
00735 }
00736 
00737 /// Check if expanding this expression is likely to incur significant cost. This
00738 /// is tricky because SCEV doesn't track which expressions are actually computed
00739 /// by the current IR.
00740 ///
00741 /// We currently allow expansion of IV increments that involve adds,
00742 /// multiplication by constants, and AddRecs from existing phis.
00743 ///
00744 /// TODO: Allow UDivExpr if we can find an existing IV increment that is an
00745 /// obvious multiple of the UDivExpr.
00746 static bool isHighCostExpansion(const SCEV *S,
00747                                 SmallPtrSet<const SCEV*, 8> &Processed,
00748                                 ScalarEvolution &SE) {
00749   // Zero/One operand expressions
00750   switch (S->getSCEVType()) {
00751   case scUnknown:
00752   case scConstant:
00753     return false;
00754   case scTruncate:
00755     return isHighCostExpansion(cast<SCEVTruncateExpr>(S)->getOperand(),
00756                                Processed, SE);
00757   case scZeroExtend:
00758     return isHighCostExpansion(cast<SCEVZeroExtendExpr>(S)->getOperand(),
00759                                Processed, SE);
00760   case scSignExtend:
00761     return isHighCostExpansion(cast<SCEVSignExtendExpr>(S)->getOperand(),
00762                                Processed, SE);
00763   }
00764 
00765   if (!Processed.insert(S))
00766     return false;
00767 
00768   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
00769     for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
00770          I != E; ++I) {
00771       if (isHighCostExpansion(*I, Processed, SE))
00772         return true;
00773     }
00774     return false;
00775   }
00776 
00777   if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
00778     if (Mul->getNumOperands() == 2) {
00779       // Multiplication by a constant is ok
00780       if (isa<SCEVConstant>(Mul->getOperand(0)))
00781         return isHighCostExpansion(Mul->getOperand(1), Processed, SE);
00782 
00783       // If we have the value of one operand, check if an existing
00784       // multiplication already generates this expression.
00785       if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Mul->getOperand(1))) {
00786         Value *UVal = U->getValue();
00787         for (User *UR : UVal->users()) {
00788           // If U is a constant, it may be used by a ConstantExpr.
00789           Instruction *UI = dyn_cast<Instruction>(UR);
00790           if (UI && UI->getOpcode() == Instruction::Mul &&
00791               SE.isSCEVable(UI->getType())) {
00792             return SE.getSCEV(UI) == Mul;
00793           }
00794         }
00795       }
00796     }
00797   }
00798 
00799   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
00800     if (isExistingPhi(AR, SE))
00801       return false;
00802   }
00803 
00804   // Fow now, consider any other type of expression (div/mul/min/max) high cost.
00805   return true;
00806 }
00807 
00808 /// DeleteTriviallyDeadInstructions - If any of the instructions is the
00809 /// specified set are trivially dead, delete them and see if this makes any of
00810 /// their operands subsequently dead.
00811 static bool
00812 DeleteTriviallyDeadInstructions(SmallVectorImpl<WeakVH> &DeadInsts) {
00813   bool Changed = false;
00814 
00815   while (!DeadInsts.empty()) {
00816     Value *V = DeadInsts.pop_back_val();
00817     Instruction *I = dyn_cast_or_null<Instruction>(V);
00818 
00819     if (!I || !isInstructionTriviallyDead(I))
00820       continue;
00821 
00822     for (User::op_iterator OI = I->op_begin(), E = I->op_end(); OI != E; ++OI)
00823       if (Instruction *U = dyn_cast<Instruction>(*OI)) {
00824         *OI = nullptr;
00825         if (U->use_empty())
00826           DeadInsts.push_back(U);
00827       }
00828 
00829     I->eraseFromParent();
00830     Changed = true;
00831   }
00832 
00833   return Changed;
00834 }
00835 
00836 namespace {
00837 class LSRUse;
00838 }
00839 
00840 /// \brief Check if the addressing mode defined by \p F is completely
00841 /// folded in \p LU at isel time.
00842 /// This includes address-mode folding and special icmp tricks.
00843 /// This function returns true if \p LU can accommodate what \p F
00844 /// defines and up to 1 base + 1 scaled + offset.
00845 /// In other words, if \p F has several base registers, this function may
00846 /// still return true. Therefore, users still need to account for
00847 /// additional base registers and/or unfolded offsets to derive an
00848 /// accurate cost model.
00849 static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
00850                                  const LSRUse &LU, const Formula &F);
00851 // Get the cost of the scaling factor used in F for LU.
00852 static unsigned getScalingFactorCost(const TargetTransformInfo &TTI,
00853                                      const LSRUse &LU, const Formula &F);
00854 
00855 namespace {
00856 
00857 /// Cost - This class is used to measure and compare candidate formulae.
00858 class Cost {
00859   /// TODO: Some of these could be merged. Also, a lexical ordering
00860   /// isn't always optimal.
00861   unsigned NumRegs;
00862   unsigned AddRecCost;
00863   unsigned NumIVMuls;
00864   unsigned NumBaseAdds;
00865   unsigned ImmCost;
00866   unsigned SetupCost;
00867   unsigned ScaleCost;
00868 
00869 public:
00870   Cost()
00871     : NumRegs(0), AddRecCost(0), NumIVMuls(0), NumBaseAdds(0), ImmCost(0),
00872       SetupCost(0), ScaleCost(0) {}
00873 
00874   bool operator<(const Cost &Other) const;
00875 
00876   void Lose();
00877 
00878 #ifndef NDEBUG
00879   // Once any of the metrics loses, they must all remain losers.
00880   bool isValid() {
00881     return ((NumRegs | AddRecCost | NumIVMuls | NumBaseAdds
00882              | ImmCost | SetupCost | ScaleCost) != ~0u)
00883       || ((NumRegs & AddRecCost & NumIVMuls & NumBaseAdds
00884            & ImmCost & SetupCost & ScaleCost) == ~0u);
00885   }
00886 #endif
00887 
00888   bool isLoser() {
00889     assert(isValid() && "invalid cost");
00890     return NumRegs == ~0u;
00891   }
00892 
00893   void RateFormula(const TargetTransformInfo &TTI,
00894                    const Formula &F,
00895                    SmallPtrSet<const SCEV *, 16> &Regs,
00896                    const DenseSet<const SCEV *> &VisitedRegs,
00897                    const Loop *L,
00898                    const SmallVectorImpl<int64_t> &Offsets,
00899                    ScalarEvolution &SE, DominatorTree &DT,
00900                    const LSRUse &LU,
00901                    SmallPtrSet<const SCEV *, 16> *LoserRegs = nullptr);
00902 
00903   void print(raw_ostream &OS) const;
00904   void dump() const;
00905 
00906 private:
00907   void RateRegister(const SCEV *Reg,
00908                     SmallPtrSet<const SCEV *, 16> &Regs,
00909                     const Loop *L,
00910                     ScalarEvolution &SE, DominatorTree &DT);
00911   void RatePrimaryRegister(const SCEV *Reg,
00912                            SmallPtrSet<const SCEV *, 16> &Regs,
00913                            const Loop *L,
00914                            ScalarEvolution &SE, DominatorTree &DT,
00915                            SmallPtrSet<const SCEV *, 16> *LoserRegs);
00916 };
00917 
00918 }
00919 
00920 /// RateRegister - Tally up interesting quantities from the given register.
00921 void Cost::RateRegister(const SCEV *Reg,
00922                         SmallPtrSet<const SCEV *, 16> &Regs,
00923                         const Loop *L,
00924                         ScalarEvolution &SE, DominatorTree &DT) {
00925   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Reg)) {
00926     // If this is an addrec for another loop, don't second-guess its addrec phi
00927     // nodes. LSR isn't currently smart enough to reason about more than one
00928     // loop at a time. LSR has already run on inner loops, will not run on outer
00929     // loops, and cannot be expected to change sibling loops.
00930     if (AR->getLoop() != L) {
00931       // If the AddRec exists, consider it's register free and leave it alone.
00932       if (isExistingPhi(AR, SE))
00933         return;
00934 
00935       // Otherwise, do not consider this formula at all.
00936       Lose();
00937       return;
00938     }
00939     AddRecCost += 1; /// TODO: This should be a function of the stride.
00940 
00941     // Add the step value register, if it needs one.
00942     // TODO: The non-affine case isn't precisely modeled here.
00943     if (!AR->isAffine() || !isa<SCEVConstant>(AR->getOperand(1))) {
00944       if (!Regs.count(AR->getOperand(1))) {
00945         RateRegister(AR->getOperand(1), Regs, L, SE, DT);
00946         if (isLoser())
00947           return;
00948       }
00949     }
00950   }
00951   ++NumRegs;
00952 
00953   // Rough heuristic; favor registers which don't require extra setup
00954   // instructions in the preheader.
00955   if (!isa<SCEVUnknown>(Reg) &&
00956       !isa<SCEVConstant>(Reg) &&
00957       !(isa<SCEVAddRecExpr>(Reg) &&
00958         (isa<SCEVUnknown>(cast<SCEVAddRecExpr>(Reg)->getStart()) ||
00959          isa<SCEVConstant>(cast<SCEVAddRecExpr>(Reg)->getStart()))))
00960     ++SetupCost;
00961 
00962     NumIVMuls += isa<SCEVMulExpr>(Reg) &&
00963                  SE.hasComputableLoopEvolution(Reg, L);
00964 }
00965 
00966 /// RatePrimaryRegister - Record this register in the set. If we haven't seen it
00967 /// before, rate it. Optional LoserRegs provides a way to declare any formula
00968 /// that refers to one of those regs an instant loser.
00969 void Cost::RatePrimaryRegister(const SCEV *Reg,
00970                                SmallPtrSet<const SCEV *, 16> &Regs,
00971                                const Loop *L,
00972                                ScalarEvolution &SE, DominatorTree &DT,
00973                                SmallPtrSet<const SCEV *, 16> *LoserRegs) {
00974   if (LoserRegs && LoserRegs->count(Reg)) {
00975     Lose();
00976     return;
00977   }
00978   if (Regs.insert(Reg)) {
00979     RateRegister(Reg, Regs, L, SE, DT);
00980     if (LoserRegs && isLoser())
00981       LoserRegs->insert(Reg);
00982   }
00983 }
00984 
00985 void Cost::RateFormula(const TargetTransformInfo &TTI,
00986                        const Formula &F,
00987                        SmallPtrSet<const SCEV *, 16> &Regs,
00988                        const DenseSet<const SCEV *> &VisitedRegs,
00989                        const Loop *L,
00990                        const SmallVectorImpl<int64_t> &Offsets,
00991                        ScalarEvolution &SE, DominatorTree &DT,
00992                        const LSRUse &LU,
00993                        SmallPtrSet<const SCEV *, 16> *LoserRegs) {
00994   assert(F.isCanonical() && "Cost is accurate only for canonical formula");
00995   // Tally up the registers.
00996   if (const SCEV *ScaledReg = F.ScaledReg) {
00997     if (VisitedRegs.count(ScaledReg)) {
00998       Lose();
00999       return;
01000     }
01001     RatePrimaryRegister(ScaledReg, Regs, L, SE, DT, LoserRegs);
01002     if (isLoser())
01003       return;
01004   }
01005   for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(),
01006        E = F.BaseRegs.end(); I != E; ++I) {
01007     const SCEV *BaseReg = *I;
01008     if (VisitedRegs.count(BaseReg)) {
01009       Lose();
01010       return;
01011     }
01012     RatePrimaryRegister(BaseReg, Regs, L, SE, DT, LoserRegs);
01013     if (isLoser())
01014       return;
01015   }
01016 
01017   // Determine how many (unfolded) adds we'll need inside the loop.
01018   size_t NumBaseParts = F.getNumRegs();
01019   if (NumBaseParts > 1)
01020     // Do not count the base and a possible second register if the target
01021     // allows to fold 2 registers.
01022     NumBaseAdds +=
01023         NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(TTI, LU, F)));
01024   NumBaseAdds += (F.UnfoldedOffset != 0);
01025 
01026   // Accumulate non-free scaling amounts.
01027   ScaleCost += getScalingFactorCost(TTI, LU, F);
01028 
01029   // Tally up the non-zero immediates.
01030   for (SmallVectorImpl<int64_t>::const_iterator I = Offsets.begin(),
01031        E = Offsets.end(); I != E; ++I) {
01032     int64_t Offset = (uint64_t)*I + F.BaseOffset;
01033     if (F.BaseGV)
01034       ImmCost += 64; // Handle symbolic values conservatively.
01035                      // TODO: This should probably be the pointer size.
01036     else if (Offset != 0)
01037       ImmCost += APInt(64, Offset, true).getMinSignedBits();
01038   }
01039   assert(isValid() && "invalid cost");
01040 }
01041 
01042 /// Lose - Set this cost to a losing value.
01043 void Cost::Lose() {
01044   NumRegs = ~0u;
01045   AddRecCost = ~0u;
01046   NumIVMuls = ~0u;
01047   NumBaseAdds = ~0u;
01048   ImmCost = ~0u;
01049   SetupCost = ~0u;
01050   ScaleCost = ~0u;
01051 }
01052 
01053 /// operator< - Choose the lower cost.
01054 bool Cost::operator<(const Cost &Other) const {
01055   return std::tie(NumRegs, AddRecCost, NumIVMuls, NumBaseAdds, ScaleCost,
01056                   ImmCost, SetupCost) <
01057          std::tie(Other.NumRegs, Other.AddRecCost, Other.NumIVMuls,
01058                   Other.NumBaseAdds, Other.ScaleCost, Other.ImmCost,
01059                   Other.SetupCost);
01060 }
01061 
01062 void Cost::print(raw_ostream &OS) const {
01063   OS << NumRegs << " reg" << (NumRegs == 1 ? "" : "s");
01064   if (AddRecCost != 0)
01065     OS << ", with addrec cost " << AddRecCost;
01066   if (NumIVMuls != 0)
01067     OS << ", plus " << NumIVMuls << " IV mul" << (NumIVMuls == 1 ? "" : "s");
01068   if (NumBaseAdds != 0)
01069     OS << ", plus " << NumBaseAdds << " base add"
01070        << (NumBaseAdds == 1 ? "" : "s");
01071   if (ScaleCost != 0)
01072     OS << ", plus " << ScaleCost << " scale cost";
01073   if (ImmCost != 0)
01074     OS << ", plus " << ImmCost << " imm cost";
01075   if (SetupCost != 0)
01076     OS << ", plus " << SetupCost << " setup cost";
01077 }
01078 
01079 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
01080 void Cost::dump() const {
01081   print(errs()); errs() << '\n';
01082 }
01083 #endif
01084 
01085 namespace {
01086 
01087 /// LSRFixup - An operand value in an instruction which is to be replaced
01088 /// with some equivalent, possibly strength-reduced, replacement.
01089 struct LSRFixup {
01090   /// UserInst - The instruction which will be updated.
01091   Instruction *UserInst;
01092 
01093   /// OperandValToReplace - The operand of the instruction which will
01094   /// be replaced. The operand may be used more than once; every instance
01095   /// will be replaced.
01096   Value *OperandValToReplace;
01097 
01098   /// PostIncLoops - If this user is to use the post-incremented value of an
01099   /// induction variable, this variable is non-null and holds the loop
01100   /// associated with the induction variable.
01101   PostIncLoopSet PostIncLoops;
01102 
01103   /// LUIdx - The index of the LSRUse describing the expression which
01104   /// this fixup needs, minus an offset (below).
01105   size_t LUIdx;
01106 
01107   /// Offset - A constant offset to be added to the LSRUse expression.
01108   /// This allows multiple fixups to share the same LSRUse with different
01109   /// offsets, for example in an unrolled loop.
01110   int64_t Offset;
01111 
01112   bool isUseFullyOutsideLoop(const Loop *L) const;
01113 
01114   LSRFixup();
01115 
01116   void print(raw_ostream &OS) const;
01117   void dump() const;
01118 };
01119 
01120 }
01121 
01122 LSRFixup::LSRFixup()
01123   : UserInst(nullptr), OperandValToReplace(nullptr), LUIdx(~size_t(0)),
01124     Offset(0) {}
01125 
01126 /// isUseFullyOutsideLoop - Test whether this fixup always uses its
01127 /// value outside of the given loop.
01128 bool LSRFixup::isUseFullyOutsideLoop(const Loop *L) const {
01129   // PHI nodes use their value in their incoming blocks.
01130   if (const PHINode *PN = dyn_cast<PHINode>(UserInst)) {
01131     for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
01132       if (PN->getIncomingValue(i) == OperandValToReplace &&
01133           L->contains(PN->getIncomingBlock(i)))
01134         return false;
01135     return true;
01136   }
01137 
01138   return !L->contains(UserInst);
01139 }
01140 
01141 void LSRFixup::print(raw_ostream &OS) const {
01142   OS << "UserInst=";
01143   // Store is common and interesting enough to be worth special-casing.
01144   if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) {
01145     OS << "store ";
01146     Store->getOperand(0)->printAsOperand(OS, /*PrintType=*/false);
01147   } else if (UserInst->getType()->isVoidTy())
01148     OS << UserInst->getOpcodeName();
01149   else
01150     UserInst->printAsOperand(OS, /*PrintType=*/false);
01151 
01152   OS << ", OperandValToReplace=";
01153   OperandValToReplace->printAsOperand(OS, /*PrintType=*/false);
01154 
01155   for (PostIncLoopSet::const_iterator I = PostIncLoops.begin(),
01156        E = PostIncLoops.end(); I != E; ++I) {
01157     OS << ", PostIncLoop=";
01158     (*I)->getHeader()->printAsOperand(OS, /*PrintType=*/false);
01159   }
01160 
01161   if (LUIdx != ~size_t(0))
01162     OS << ", LUIdx=" << LUIdx;
01163 
01164   if (Offset != 0)
01165     OS << ", Offset=" << Offset;
01166 }
01167 
01168 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
01169 void LSRFixup::dump() const {
01170   print(errs()); errs() << '\n';
01171 }
01172 #endif
01173 
01174 namespace {
01175 
01176 /// UniquifierDenseMapInfo - A DenseMapInfo implementation for holding
01177 /// DenseMaps and DenseSets of sorted SmallVectors of const SCEV*.
01178 struct UniquifierDenseMapInfo {
01179   static SmallVector<const SCEV *, 4> getEmptyKey() {
01180     SmallVector<const SCEV *, 4>  V;
01181     V.push_back(reinterpret_cast<const SCEV *>(-1));
01182     return V;
01183   }
01184 
01185   static SmallVector<const SCEV *, 4> getTombstoneKey() {
01186     SmallVector<const SCEV *, 4> V;
01187     V.push_back(reinterpret_cast<const SCEV *>(-2));
01188     return V;
01189   }
01190 
01191   static unsigned getHashValue(const SmallVector<const SCEV *, 4> &V) {
01192     return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
01193   }
01194 
01195   static bool isEqual(const SmallVector<const SCEV *, 4> &LHS,
01196                       const SmallVector<const SCEV *, 4> &RHS) {
01197     return LHS == RHS;
01198   }
01199 };
01200 
01201 /// LSRUse - This class holds the state that LSR keeps for each use in
01202 /// IVUsers, as well as uses invented by LSR itself. It includes information
01203 /// about what kinds of things can be folded into the user, information about
01204 /// the user itself, and information about how the use may be satisfied.
01205 /// TODO: Represent multiple users of the same expression in common?
01206 class LSRUse {
01207   DenseSet<SmallVector<const SCEV *, 4>, UniquifierDenseMapInfo> Uniquifier;
01208 
01209 public:
01210   /// KindType - An enum for a kind of use, indicating what types of
01211   /// scaled and immediate operands it might support.
01212   enum KindType {
01213     Basic,   ///< A normal use, with no folding.
01214     Special, ///< A special case of basic, allowing -1 scales.
01215     Address, ///< An address use; folding according to TargetLowering
01216     ICmpZero ///< An equality icmp with both operands folded into one.
01217     // TODO: Add a generic icmp too?
01218   };
01219 
01220   typedef PointerIntPair<const SCEV *, 2, KindType> SCEVUseKindPair;
01221 
01222   KindType Kind;
01223   Type *AccessTy;
01224 
01225   SmallVector<int64_t, 8> Offsets;
01226   int64_t MinOffset;
01227   int64_t MaxOffset;
01228 
01229   /// AllFixupsOutsideLoop - This records whether all of the fixups using this
01230   /// LSRUse are outside of the loop, in which case some special-case heuristics
01231   /// may be used.
01232   bool AllFixupsOutsideLoop;
01233 
01234   /// RigidFormula is set to true to guarantee that this use will be associated
01235   /// with a single formula--the one that initially matched. Some SCEV
01236   /// expressions cannot be expanded. This allows LSR to consider the registers
01237   /// used by those expressions without the need to expand them later after
01238   /// changing the formula.
01239   bool RigidFormula;
01240 
01241   /// WidestFixupType - This records the widest use type for any fixup using
01242   /// this LSRUse. FindUseWithSimilarFormula can't consider uses with different
01243   /// max fixup widths to be equivalent, because the narrower one may be relying
01244   /// on the implicit truncation to truncate away bogus bits.
01245   Type *WidestFixupType;
01246 
01247   /// Formulae - A list of ways to build a value that can satisfy this user.
01248   /// After the list is populated, one of these is selected heuristically and
01249   /// used to formulate a replacement for OperandValToReplace in UserInst.
01250   SmallVector<Formula, 12> Formulae;
01251 
01252   /// Regs - The set of register candidates used by all formulae in this LSRUse.
01253   SmallPtrSet<const SCEV *, 4> Regs;
01254 
01255   LSRUse(KindType K, Type *T) : Kind(K), AccessTy(T),
01256                                       MinOffset(INT64_MAX),
01257                                       MaxOffset(INT64_MIN),
01258                                       AllFixupsOutsideLoop(true),
01259                                       RigidFormula(false),
01260                                       WidestFixupType(nullptr) {}
01261 
01262   bool HasFormulaWithSameRegs(const Formula &F) const;
01263   bool InsertFormula(const Formula &F);
01264   void DeleteFormula(Formula &F);
01265   void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses);
01266 
01267   void print(raw_ostream &OS) const;
01268   void dump() const;
01269 };
01270 
01271 }
01272 
01273 /// HasFormula - Test whether this use as a formula which has the same
01274 /// registers as the given formula.
01275 bool LSRUse::HasFormulaWithSameRegs(const Formula &F) const {
01276   SmallVector<const SCEV *, 4> Key = F.BaseRegs;
01277   if (F.ScaledReg) Key.push_back(F.ScaledReg);
01278   // Unstable sort by host order ok, because this is only used for uniquifying.
01279   std::sort(Key.begin(), Key.end());
01280   return Uniquifier.count(Key);
01281 }
01282 
01283 /// InsertFormula - If the given formula has not yet been inserted, add it to
01284 /// the list, and return true. Return false otherwise.
01285 /// The formula must be in canonical form.
01286 bool LSRUse::InsertFormula(const Formula &F) {
01287   assert(F.isCanonical() && "Invalid canonical representation");
01288 
01289   if (!Formulae.empty() && RigidFormula)
01290     return false;
01291 
01292   SmallVector<const SCEV *, 4> Key = F.BaseRegs;
01293   if (F.ScaledReg) Key.push_back(F.ScaledReg);
01294   // Unstable sort by host order ok, because this is only used for uniquifying.
01295   std::sort(Key.begin(), Key.end());
01296 
01297   if (!Uniquifier.insert(Key).second)
01298     return false;
01299 
01300   // Using a register to hold the value of 0 is not profitable.
01301   assert((!F.ScaledReg || !F.ScaledReg->isZero()) &&
01302          "Zero allocated in a scaled register!");
01303 #ifndef NDEBUG
01304   for (SmallVectorImpl<const SCEV *>::const_iterator I =
01305        F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I)
01306     assert(!(*I)->isZero() && "Zero allocated in a base register!");
01307 #endif
01308 
01309   // Add the formula to the list.
01310   Formulae.push_back(F);
01311 
01312   // Record registers now being used by this use.
01313   Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end());
01314   if (F.ScaledReg)
01315     Regs.insert(F.ScaledReg);
01316 
01317   return true;
01318 }
01319 
01320 /// DeleteFormula - Remove the given formula from this use's list.
01321 void LSRUse::DeleteFormula(Formula &F) {
01322   if (&F != &Formulae.back())
01323     std::swap(F, Formulae.back());
01324   Formulae.pop_back();
01325 }
01326 
01327 /// RecomputeRegs - Recompute the Regs field, and update RegUses.
01328 void LSRUse::RecomputeRegs(size_t LUIdx, RegUseTracker &RegUses) {
01329   // Now that we've filtered out some formulae, recompute the Regs set.
01330   SmallPtrSet<const SCEV *, 4> OldRegs = Regs;
01331   Regs.clear();
01332   for (SmallVectorImpl<Formula>::const_iterator I = Formulae.begin(),
01333        E = Formulae.end(); I != E; ++I) {
01334     const Formula &F = *I;
01335     if (F.ScaledReg) Regs.insert(F.ScaledReg);
01336     Regs.insert(F.BaseRegs.begin(), F.BaseRegs.end());
01337   }
01338 
01339   // Update the RegTracker.
01340   for (SmallPtrSet<const SCEV *, 4>::iterator I = OldRegs.begin(),
01341        E = OldRegs.end(); I != E; ++I)
01342     if (!Regs.count(*I))
01343       RegUses.DropRegister(*I, LUIdx);
01344 }
01345 
01346 void LSRUse::print(raw_ostream &OS) const {
01347   OS << "LSR Use: Kind=";
01348   switch (Kind) {
01349   case Basic:    OS << "Basic"; break;
01350   case Special:  OS << "Special"; break;
01351   case ICmpZero: OS << "ICmpZero"; break;
01352   case Address:
01353     OS << "Address of ";
01354     if (AccessTy->isPointerTy())
01355       OS << "pointer"; // the full pointer type could be really verbose
01356     else
01357       OS << *AccessTy;
01358   }
01359 
01360   OS << ", Offsets={";
01361   for (SmallVectorImpl<int64_t>::const_iterator I = Offsets.begin(),
01362        E = Offsets.end(); I != E; ++I) {
01363     OS << *I;
01364     if (std::next(I) != E)
01365       OS << ',';
01366   }
01367   OS << '}';
01368 
01369   if (AllFixupsOutsideLoop)
01370     OS << ", all-fixups-outside-loop";
01371 
01372   if (WidestFixupType)
01373     OS << ", widest fixup type: " << *WidestFixupType;
01374 }
01375 
01376 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
01377 void LSRUse::dump() const {
01378   print(errs()); errs() << '\n';
01379 }
01380 #endif
01381 
01382 static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
01383                                  LSRUse::KindType Kind, Type *AccessTy,
01384                                  GlobalValue *BaseGV, int64_t BaseOffset,
01385                                  bool HasBaseReg, int64_t Scale) {
01386   switch (Kind) {
01387   case LSRUse::Address:
01388     return TTI.isLegalAddressingMode(AccessTy, BaseGV, BaseOffset, HasBaseReg, Scale);
01389 
01390     // Otherwise, just guess that reg+reg addressing is legal.
01391     //return ;
01392 
01393   case LSRUse::ICmpZero:
01394     // There's not even a target hook for querying whether it would be legal to
01395     // fold a GV into an ICmp.
01396     if (BaseGV)
01397       return false;
01398 
01399     // ICmp only has two operands; don't allow more than two non-trivial parts.
01400     if (Scale != 0 && HasBaseReg && BaseOffset != 0)
01401       return false;
01402 
01403     // ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale by
01404     // putting the scaled register in the other operand of the icmp.
01405     if (Scale != 0 && Scale != -1)
01406       return false;
01407 
01408     // If we have low-level target information, ask the target if it can fold an
01409     // integer immediate on an icmp.
01410     if (BaseOffset != 0) {
01411       // We have one of:
01412       // ICmpZero     BaseReg + BaseOffset => ICmp BaseReg, -BaseOffset
01413       // ICmpZero -1*ScaleReg + BaseOffset => ICmp ScaleReg, BaseOffset
01414       // Offs is the ICmp immediate.
01415       if (Scale == 0)
01416         // The cast does the right thing with INT64_MIN.
01417         BaseOffset = -(uint64_t)BaseOffset;
01418       return TTI.isLegalICmpImmediate(BaseOffset);
01419     }
01420 
01421     // ICmpZero BaseReg + -1*ScaleReg => ICmp BaseReg, ScaleReg
01422     return true;
01423 
01424   case LSRUse::Basic:
01425     // Only handle single-register values.
01426     return !BaseGV && Scale == 0 && BaseOffset == 0;
01427 
01428   case LSRUse::Special:
01429     // Special case Basic to handle -1 scales.
01430     return !BaseGV && (Scale == 0 || Scale == -1) && BaseOffset == 0;
01431   }
01432 
01433   llvm_unreachable("Invalid LSRUse Kind!");
01434 }
01435 
01436 static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
01437                                  int64_t MinOffset, int64_t MaxOffset,
01438                                  LSRUse::KindType Kind, Type *AccessTy,
01439                                  GlobalValue *BaseGV, int64_t BaseOffset,
01440                                  bool HasBaseReg, int64_t Scale) {
01441   // Check for overflow.
01442   if (((int64_t)((uint64_t)BaseOffset + MinOffset) > BaseOffset) !=
01443       (MinOffset > 0))
01444     return false;
01445   MinOffset = (uint64_t)BaseOffset + MinOffset;
01446   if (((int64_t)((uint64_t)BaseOffset + MaxOffset) > BaseOffset) !=
01447       (MaxOffset > 0))
01448     return false;
01449   MaxOffset = (uint64_t)BaseOffset + MaxOffset;
01450 
01451   return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, MinOffset,
01452                               HasBaseReg, Scale) &&
01453          isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, MaxOffset,
01454                               HasBaseReg, Scale);
01455 }
01456 
01457 static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
01458                                  int64_t MinOffset, int64_t MaxOffset,
01459                                  LSRUse::KindType Kind, Type *AccessTy,
01460                                  const Formula &F) {
01461   // For the purpose of isAMCompletelyFolded either having a canonical formula
01462   // or a scale not equal to zero is correct.
01463   // Problems may arise from non canonical formulae having a scale == 0.
01464   // Strictly speaking it would best to just rely on canonical formulae.
01465   // However, when we generate the scaled formulae, we first check that the
01466   // scaling factor is profitable before computing the actual ScaledReg for
01467   // compile time sake.
01468   assert((F.isCanonical() || F.Scale != 0));
01469   return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
01470                               F.BaseGV, F.BaseOffset, F.HasBaseReg, F.Scale);
01471 }
01472 
01473 /// isLegalUse - Test whether we know how to expand the current formula.
01474 static bool isLegalUse(const TargetTransformInfo &TTI, int64_t MinOffset,
01475                        int64_t MaxOffset, LSRUse::KindType Kind, Type *AccessTy,
01476                        GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg,
01477                        int64_t Scale) {
01478   // We know how to expand completely foldable formulae.
01479   return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
01480                               BaseOffset, HasBaseReg, Scale) ||
01481          // Or formulae that use a base register produced by a sum of base
01482          // registers.
01483          (Scale == 1 &&
01484           isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
01485                                BaseGV, BaseOffset, true, 0));
01486 }
01487 
01488 static bool isLegalUse(const TargetTransformInfo &TTI, int64_t MinOffset,
01489                        int64_t MaxOffset, LSRUse::KindType Kind, Type *AccessTy,
01490                        const Formula &F) {
01491   return isLegalUse(TTI, MinOffset, MaxOffset, Kind, AccessTy, F.BaseGV,
01492                     F.BaseOffset, F.HasBaseReg, F.Scale);
01493 }
01494 
01495 static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
01496                                  const LSRUse &LU, const Formula &F) {
01497   return isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
01498                               LU.AccessTy, F.BaseGV, F.BaseOffset, F.HasBaseReg,
01499                               F.Scale);
01500 }
01501 
01502 static unsigned getScalingFactorCost(const TargetTransformInfo &TTI,
01503                                      const LSRUse &LU, const Formula &F) {
01504   if (!F.Scale)
01505     return 0;
01506 
01507   // If the use is not completely folded in that instruction, we will have to
01508   // pay an extra cost only for scale != 1.
01509   if (!isAMCompletelyFolded(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind,
01510                             LU.AccessTy, F))
01511     return F.Scale != 1;
01512 
01513   switch (LU.Kind) {
01514   case LSRUse::Address: {
01515     // Check the scaling factor cost with both the min and max offsets.
01516     int ScaleCostMinOffset =
01517       TTI.getScalingFactorCost(LU.AccessTy, F.BaseGV,
01518                                F.BaseOffset + LU.MinOffset,
01519                                F.HasBaseReg, F.Scale);
01520     int ScaleCostMaxOffset =
01521       TTI.getScalingFactorCost(LU.AccessTy, F.BaseGV,
01522                                F.BaseOffset + LU.MaxOffset,
01523                                F.HasBaseReg, F.Scale);
01524 
01525     assert(ScaleCostMinOffset >= 0 && ScaleCostMaxOffset >= 0 &&
01526            "Legal addressing mode has an illegal cost!");
01527     return std::max(ScaleCostMinOffset, ScaleCostMaxOffset);
01528   }
01529   case LSRUse::ICmpZero:
01530   case LSRUse::Basic:
01531   case LSRUse::Special:
01532     // The use is completely folded, i.e., everything is folded into the
01533     // instruction.
01534     return 0;
01535   }
01536 
01537   llvm_unreachable("Invalid LSRUse Kind!");
01538 }
01539 
01540 static bool isAlwaysFoldable(const TargetTransformInfo &TTI,
01541                              LSRUse::KindType Kind, Type *AccessTy,
01542                              GlobalValue *BaseGV, int64_t BaseOffset,
01543                              bool HasBaseReg) {
01544   // Fast-path: zero is always foldable.
01545   if (BaseOffset == 0 && !BaseGV) return true;
01546 
01547   // Conservatively, create an address with an immediate and a
01548   // base and a scale.
01549   int64_t Scale = Kind == LSRUse::ICmpZero ? -1 : 1;
01550 
01551   // Canonicalize a scale of 1 to a base register if the formula doesn't
01552   // already have a base register.
01553   if (!HasBaseReg && Scale == 1) {
01554     Scale = 0;
01555     HasBaseReg = true;
01556   }
01557 
01558   return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset,
01559                               HasBaseReg, Scale);
01560 }
01561 
01562 static bool isAlwaysFoldable(const TargetTransformInfo &TTI,
01563                              ScalarEvolution &SE, int64_t MinOffset,
01564                              int64_t MaxOffset, LSRUse::KindType Kind,
01565                              Type *AccessTy, const SCEV *S, bool HasBaseReg) {
01566   // Fast-path: zero is always foldable.
01567   if (S->isZero()) return true;
01568 
01569   // Conservatively, create an address with an immediate and a
01570   // base and a scale.
01571   int64_t BaseOffset = ExtractImmediate(S, SE);
01572   GlobalValue *BaseGV = ExtractSymbol(S, SE);
01573 
01574   // If there's anything else involved, it's not foldable.
01575   if (!S->isZero()) return false;
01576 
01577   // Fast-path: zero is always foldable.
01578   if (BaseOffset == 0 && !BaseGV) return true;
01579 
01580   // Conservatively, create an address with an immediate and a
01581   // base and a scale.
01582   int64_t Scale = Kind == LSRUse::ICmpZero ? -1 : 1;
01583 
01584   return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
01585                               BaseOffset, HasBaseReg, Scale);
01586 }
01587 
01588 namespace {
01589 
01590 /// IVInc - An individual increment in a Chain of IV increments.
01591 /// Relate an IV user to an expression that computes the IV it uses from the IV
01592 /// used by the previous link in the Chain.
01593 ///
01594 /// For the head of a chain, IncExpr holds the absolute SCEV expression for the
01595 /// original IVOperand. The head of the chain's IVOperand is only valid during
01596 /// chain collection, before LSR replaces IV users. During chain generation,
01597 /// IncExpr can be used to find the new IVOperand that computes the same
01598 /// expression.
01599 struct IVInc {
01600   Instruction *UserInst;
01601   Value* IVOperand;
01602   const SCEV *IncExpr;
01603 
01604   IVInc(Instruction *U, Value *O, const SCEV *E):
01605     UserInst(U), IVOperand(O), IncExpr(E) {}
01606 };
01607 
01608 // IVChain - The list of IV increments in program order.
01609 // We typically add the head of a chain without finding subsequent links.
01610 struct IVChain {
01611   SmallVector<IVInc,1> Incs;
01612   const SCEV *ExprBase;
01613 
01614   IVChain() : ExprBase(nullptr) {}
01615 
01616   IVChain(const IVInc &Head, const SCEV *Base)
01617     : Incs(1, Head), ExprBase(Base) {}
01618 
01619   typedef SmallVectorImpl<IVInc>::const_iterator const_iterator;
01620 
01621   // begin - return the first increment in the chain.
01622   const_iterator begin() const {
01623     assert(!Incs.empty());
01624     return std::next(Incs.begin());
01625   }
01626   const_iterator end() const {
01627     return Incs.end();
01628   }
01629 
01630   // hasIncs - Returns true if this chain contains any increments.
01631   bool hasIncs() const { return Incs.size() >= 2; }
01632 
01633   // add - Add an IVInc to the end of this chain.
01634   void add(const IVInc &X) { Incs.push_back(X); }
01635 
01636   // tailUserInst - Returns the last UserInst in the chain.
01637   Instruction *tailUserInst() const { return Incs.back().UserInst; }
01638 
01639   // isProfitableIncrement - Returns true if IncExpr can be profitably added to
01640   // this chain.
01641   bool isProfitableIncrement(const SCEV *OperExpr,
01642                              const SCEV *IncExpr,
01643                              ScalarEvolution&);
01644 };
01645 
01646 /// ChainUsers - Helper for CollectChains to track multiple IV increment uses.
01647 /// Distinguish between FarUsers that definitely cross IV increments and
01648 /// NearUsers that may be used between IV increments.
01649 struct ChainUsers {
01650   SmallPtrSet<Instruction*, 4> FarUsers;
01651   SmallPtrSet<Instruction*, 4> NearUsers;
01652 };
01653 
01654 /// LSRInstance - This class holds state for the main loop strength reduction
01655 /// logic.
01656 class LSRInstance {
01657   IVUsers &IU;
01658   ScalarEvolution &SE;
01659   DominatorTree &DT;
01660   LoopInfo &LI;
01661   const TargetTransformInfo &TTI;
01662   Loop *const L;
01663   bool Changed;
01664 
01665   /// IVIncInsertPos - This is the insert position that the current loop's
01666   /// induction variable increment should be placed. In simple loops, this is
01667   /// the latch block's terminator. But in more complicated cases, this is a
01668   /// position which will dominate all the in-loop post-increment users.
01669   Instruction *IVIncInsertPos;
01670 
01671   /// Factors - Interesting factors between use strides.
01672   SmallSetVector<int64_t, 8> Factors;
01673 
01674   /// Types - Interesting use types, to facilitate truncation reuse.
01675   SmallSetVector<Type *, 4> Types;
01676 
01677   /// Fixups - The list of operands which are to be replaced.
01678   SmallVector<LSRFixup, 16> Fixups;
01679 
01680   /// Uses - The list of interesting uses.
01681   SmallVector<LSRUse, 16> Uses;
01682 
01683   /// RegUses - Track which uses use which register candidates.
01684   RegUseTracker RegUses;
01685 
01686   // Limit the number of chains to avoid quadratic behavior. We don't expect to
01687   // have more than a few IV increment chains in a loop. Missing a Chain falls
01688   // back to normal LSR behavior for those uses.
01689   static const unsigned MaxChains = 8;
01690 
01691   /// IVChainVec - IV users can form a chain of IV increments.
01692   SmallVector<IVChain, MaxChains> IVChainVec;
01693 
01694   /// IVIncSet - IV users that belong to profitable IVChains.
01695   SmallPtrSet<Use*, MaxChains> IVIncSet;
01696 
01697   void OptimizeShadowIV();
01698   bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse);
01699   ICmpInst *OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse);
01700   void OptimizeLoopTermCond();
01701 
01702   void ChainInstruction(Instruction *UserInst, Instruction *IVOper,
01703                         SmallVectorImpl<ChainUsers> &ChainUsersVec);
01704   void FinalizeChain(IVChain &Chain);
01705   void CollectChains();
01706   void GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter,
01707                        SmallVectorImpl<WeakVH> &DeadInsts);
01708 
01709   void CollectInterestingTypesAndFactors();
01710   void CollectFixupsAndInitialFormulae();
01711 
01712   LSRFixup &getNewFixup() {
01713     Fixups.push_back(LSRFixup());
01714     return Fixups.back();
01715   }
01716 
01717   // Support for sharing of LSRUses between LSRFixups.
01718   typedef DenseMap<LSRUse::SCEVUseKindPair, size_t> UseMapTy;
01719   UseMapTy UseMap;
01720 
01721   bool reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg,
01722                           LSRUse::KindType Kind, Type *AccessTy);
01723 
01724   std::pair<size_t, int64_t> getUse(const SCEV *&Expr,
01725                                     LSRUse::KindType Kind,
01726                                     Type *AccessTy);
01727 
01728   void DeleteUse(LSRUse &LU, size_t LUIdx);
01729 
01730   LSRUse *FindUseWithSimilarFormula(const Formula &F, const LSRUse &OrigLU);
01731 
01732   void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
01733   void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
01734   void CountRegisters(const Formula &F, size_t LUIdx);
01735   bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
01736 
01737   void CollectLoopInvariantFixupsAndFormulae();
01738 
01739   void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
01740                               unsigned Depth = 0);
01741 
01742   void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
01743                                   const Formula &Base, unsigned Depth,
01744                                   size_t Idx, bool IsScaledReg = false);
01745   void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
01746   void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
01747                                    const Formula &Base, size_t Idx,
01748                                    bool IsScaledReg = false);
01749   void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
01750   void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
01751                                    const Formula &Base,
01752                                    const SmallVectorImpl<int64_t> &Worklist,
01753                                    size_t Idx, bool IsScaledReg = false);
01754   void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
01755   void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
01756   void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
01757   void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
01758   void GenerateCrossUseConstantOffsets();
01759   void GenerateAllReuseFormulae();
01760 
01761   void FilterOutUndesirableDedicatedRegisters();
01762 
01763   size_t EstimateSearchSpaceComplexity() const;
01764   void NarrowSearchSpaceByDetectingSupersets();
01765   void NarrowSearchSpaceByCollapsingUnrolledCode();
01766   void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
01767   void NarrowSearchSpaceByPickingWinnerRegs();
01768   void NarrowSearchSpaceUsingHeuristics();
01769 
01770   void SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
01771                     Cost &SolutionCost,
01772                     SmallVectorImpl<const Formula *> &Workspace,
01773                     const Cost &CurCost,
01774                     const SmallPtrSet<const SCEV *, 16> &CurRegs,
01775                     DenseSet<const SCEV *> &VisitedRegs) const;
01776   void Solve(SmallVectorImpl<const Formula *> &Solution) const;
01777 
01778   BasicBlock::iterator
01779     HoistInsertPosition(BasicBlock::iterator IP,
01780                         const SmallVectorImpl<Instruction *> &Inputs) const;
01781   BasicBlock::iterator
01782     AdjustInsertPositionForExpand(BasicBlock::iterator IP,
01783                                   const LSRFixup &LF,
01784                                   const LSRUse &LU,
01785                                   SCEVExpander &Rewriter) const;
01786 
01787   Value *Expand(const LSRFixup &LF,
01788                 const Formula &F,
01789                 BasicBlock::iterator IP,
01790                 SCEVExpander &Rewriter,
01791                 SmallVectorImpl<WeakVH> &DeadInsts) const;
01792   void RewriteForPHI(PHINode *PN, const LSRFixup &LF,
01793                      const Formula &F,
01794                      SCEVExpander &Rewriter,
01795                      SmallVectorImpl<WeakVH> &DeadInsts,
01796                      Pass *P) const;
01797   void Rewrite(const LSRFixup &LF,
01798                const Formula &F,
01799                SCEVExpander &Rewriter,
01800                SmallVectorImpl<WeakVH> &DeadInsts,
01801                Pass *P) const;
01802   void ImplementSolution(const SmallVectorImpl<const Formula *> &Solution,
01803                          Pass *P);
01804 
01805 public:
01806   LSRInstance(Loop *L, Pass *P);
01807 
01808   bool getChanged() const { return Changed; }
01809 
01810   void print_factors_and_types(raw_ostream &OS) const;
01811   void print_fixups(raw_ostream &OS) const;
01812   void print_uses(raw_ostream &OS) const;
01813   void print(raw_ostream &OS) const;
01814   void dump() const;
01815 };
01816 
01817 }
01818 
01819 /// OptimizeShadowIV - If IV is used in a int-to-float cast
01820 /// inside the loop then try to eliminate the cast operation.
01821 void LSRInstance::OptimizeShadowIV() {
01822   const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
01823   if (isa<SCEVCouldNotCompute>(BackedgeTakenCount))
01824     return;
01825 
01826   for (IVUsers::const_iterator UI = IU.begin(), E = IU.end();
01827        UI != E; /* empty */) {
01828     IVUsers::const_iterator CandidateUI = UI;
01829     ++UI;
01830     Instruction *ShadowUse = CandidateUI->getUser();
01831     Type *DestTy = nullptr;
01832     bool IsSigned = false;
01833 
01834     /* If shadow use is a int->float cast then insert a second IV
01835        to eliminate this cast.
01836 
01837          for (unsigned i = 0; i < n; ++i)
01838            foo((double)i);
01839 
01840        is transformed into
01841 
01842          double d = 0.0;
01843          for (unsigned i = 0; i < n; ++i, ++d)
01844            foo(d);
01845     */
01846     if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(CandidateUI->getUser())) {
01847       IsSigned = false;
01848       DestTy = UCast->getDestTy();
01849     }
01850     else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(CandidateUI->getUser())) {
01851       IsSigned = true;
01852       DestTy = SCast->getDestTy();
01853     }
01854     if (!DestTy) continue;
01855 
01856     // If target does not support DestTy natively then do not apply
01857     // this transformation.
01858     if (!TTI.isTypeLegal(DestTy)) continue;
01859 
01860     PHINode *PH = dyn_cast<PHINode>(ShadowUse->getOperand(0));
01861     if (!PH) continue;
01862     if (PH->getNumIncomingValues() != 2) continue;
01863 
01864     Type *SrcTy = PH->getType();
01865     int Mantissa = DestTy->getFPMantissaWidth();
01866     if (Mantissa == -1) continue;
01867     if ((int)SE.getTypeSizeInBits(SrcTy) > Mantissa)
01868       continue;
01869 
01870     unsigned Entry, Latch;
01871     if (PH->getIncomingBlock(0) == L->getLoopPreheader()) {
01872       Entry = 0;
01873       Latch = 1;
01874     } else {
01875       Entry = 1;
01876       Latch = 0;
01877     }
01878 
01879     ConstantInt *Init = dyn_cast<ConstantInt>(PH->getIncomingValue(Entry));
01880     if (!Init) continue;
01881     Constant *NewInit = ConstantFP::get(DestTy, IsSigned ?
01882                                         (double)Init->getSExtValue() :
01883                                         (double)Init->getZExtValue());
01884 
01885     BinaryOperator *Incr =
01886       dyn_cast<BinaryOperator>(PH->getIncomingValue(Latch));
01887     if (!Incr) continue;
01888     if (Incr->getOpcode() != Instruction::Add
01889         && Incr->getOpcode() != Instruction::Sub)
01890       continue;
01891 
01892     /* Initialize new IV, double d = 0.0 in above example. */
01893     ConstantInt *C = nullptr;
01894     if (Incr->getOperand(0) == PH)
01895       C = dyn_cast<ConstantInt>(Incr->getOperand(1));
01896     else if (Incr->getOperand(1) == PH)
01897       C = dyn_cast<ConstantInt>(Incr->getOperand(0));
01898     else
01899       continue;
01900 
01901     if (!C) continue;
01902 
01903     // Ignore negative constants, as the code below doesn't handle them
01904     // correctly. TODO: Remove this restriction.
01905     if (!C->getValue().isStrictlyPositive()) continue;
01906 
01907     /* Add new PHINode. */
01908     PHINode *NewPH = PHINode::Create(DestTy, 2, "IV.S.", PH);
01909 
01910     /* create new increment. '++d' in above example. */
01911     Constant *CFP = ConstantFP::get(DestTy, C->getZExtValue());
01912     BinaryOperator *NewIncr =
01913       BinaryOperator::Create(Incr->getOpcode() == Instruction::Add ?
01914                                Instruction::FAdd : Instruction::FSub,
01915                              NewPH, CFP, "IV.S.next.", Incr);
01916 
01917     NewPH->addIncoming(NewInit, PH->getIncomingBlock(Entry));
01918     NewPH->addIncoming(NewIncr, PH->getIncomingBlock(Latch));
01919 
01920     /* Remove cast operation */
01921     ShadowUse->replaceAllUsesWith(NewPH);
01922     ShadowUse->eraseFromParent();
01923     Changed = true;
01924     break;
01925   }
01926 }
01927 
01928 /// FindIVUserForCond - If Cond has an operand that is an expression of an IV,
01929 /// set the IV user and stride information and return true, otherwise return
01930 /// false.
01931 bool LSRInstance::FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse) {
01932   for (IVUsers::iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI)
01933     if (UI->getUser() == Cond) {
01934       // NOTE: we could handle setcc instructions with multiple uses here, but
01935       // InstCombine does it as well for simple uses, it's not clear that it
01936       // occurs enough in real life to handle.
01937       CondUse = UI;
01938       return true;
01939     }
01940   return false;
01941 }
01942 
01943 /// OptimizeMax - Rewrite the loop's terminating condition if it uses
01944 /// a max computation.
01945 ///
01946 /// This is a narrow solution to a specific, but acute, problem. For loops
01947 /// like this:
01948 ///
01949 ///   i = 0;
01950 ///   do {
01951 ///     p[i] = 0.0;
01952 ///   } while (++i < n);
01953 ///
01954 /// the trip count isn't just 'n', because 'n' might not be positive. And
01955 /// unfortunately this can come up even for loops where the user didn't use
01956 /// a C do-while loop. For example, seemingly well-behaved top-test loops
01957 /// will commonly be lowered like this:
01958 //
01959 ///   if (n > 0) {
01960 ///     i = 0;
01961 ///     do {
01962 ///       p[i] = 0.0;
01963 ///     } while (++i < n);
01964 ///   }
01965 ///
01966 /// and then it's possible for subsequent optimization to obscure the if
01967 /// test in such a way that indvars can't find it.
01968 ///
01969 /// When indvars can't find the if test in loops like this, it creates a
01970 /// max expression, which allows it to give the loop a canonical
01971 /// induction variable:
01972 ///
01973 ///   i = 0;
01974 ///   max = n < 1 ? 1 : n;
01975 ///   do {
01976 ///     p[i] = 0.0;
01977 ///   } while (++i != max);
01978 ///
01979 /// Canonical induction variables are necessary because the loop passes
01980 /// are designed around them. The most obvious example of this is the
01981 /// LoopInfo analysis, which doesn't remember trip count values. It
01982 /// expects to be able to rediscover the trip count each time it is
01983 /// needed, and it does this using a simple analysis that only succeeds if
01984 /// the loop has a canonical induction variable.
01985 ///
01986 /// However, when it comes time to generate code, the maximum operation
01987 /// can be quite costly, especially if it's inside of an outer loop.
01988 ///
01989 /// This function solves this problem by detecting this type of loop and
01990 /// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting
01991 /// the instructions for the maximum computation.
01992 ///
01993 ICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) {
01994   // Check that the loop matches the pattern we're looking for.
01995   if (Cond->getPredicate() != CmpInst::ICMP_EQ &&
01996       Cond->getPredicate() != CmpInst::ICMP_NE)
01997     return Cond;
01998 
01999   SelectInst *Sel = dyn_cast<SelectInst>(Cond->getOperand(1));
02000   if (!Sel || !Sel->hasOneUse()) return Cond;
02001 
02002   const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
02003   if (isa<SCEVCouldNotCompute>(BackedgeTakenCount))
02004     return Cond;
02005   const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1);
02006 
02007   // Add one to the backedge-taken count to get the trip count.
02008   const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount);
02009   if (IterationCount != SE.getSCEV(Sel)) return Cond;
02010 
02011   // Check for a max calculation that matches the pattern. There's no check
02012   // for ICMP_ULE here because the comparison would be with zero, which
02013   // isn't interesting.
02014   CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
02015   const SCEVNAryExpr *Max = nullptr;
02016   if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(BackedgeTakenCount)) {
02017     Pred = ICmpInst::ICMP_SLE;
02018     Max = S;
02019   } else if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(IterationCount)) {
02020     Pred = ICmpInst::ICMP_SLT;
02021     Max = S;
02022   } else if (const SCEVUMaxExpr *U = dyn_cast<SCEVUMaxExpr>(IterationCount)) {
02023     Pred = ICmpInst::ICMP_ULT;
02024     Max = U;
02025   } else {
02026     // No match; bail.
02027     return Cond;
02028   }
02029 
02030   // To handle a max with more than two operands, this optimization would
02031   // require additional checking and setup.
02032   if (Max->getNumOperands() != 2)
02033     return Cond;
02034 
02035   const SCEV *MaxLHS = Max->getOperand(0);
02036   const SCEV *MaxRHS = Max->getOperand(1);
02037 
02038   // ScalarEvolution canonicalizes constants to the left. For < and >, look
02039   // for a comparison with 1. For <= and >=, a comparison with zero.
02040   if (!MaxLHS ||
02041       (ICmpInst::isTrueWhenEqual(Pred) ? !MaxLHS->isZero() : (MaxLHS != One)))
02042     return Cond;
02043 
02044   // Check the relevant induction variable for conformance to
02045   // the pattern.
02046   const SCEV *IV = SE.getSCEV(Cond->getOperand(0));
02047   const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(IV);
02048   if (!AR || !AR->isAffine() ||
02049       AR->getStart() != One ||
02050       AR->getStepRecurrence(SE) != One)
02051     return Cond;
02052 
02053   assert(AR->getLoop() == L &&
02054          "Loop condition operand is an addrec in a different loop!");
02055 
02056   // Check the right operand of the select, and remember it, as it will
02057   // be used in the new comparison instruction.
02058   Value *NewRHS = nullptr;
02059   if (ICmpInst::isTrueWhenEqual(Pred)) {
02060     // Look for n+1, and grab n.
02061     if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(1)))
02062       if (ConstantInt *BO1 = dyn_cast<ConstantInt>(BO->getOperand(1)))
02063          if (BO1->isOne() && SE.getSCEV(BO->getOperand(0)) == MaxRHS)
02064            NewRHS = BO->getOperand(0);
02065     if (AddOperator *BO = dyn_cast<AddOperator>(Sel->getOperand(2)))
02066       if (ConstantInt *BO1 = dyn_cast<ConstantInt>(BO->getOperand(1)))
02067         if (BO1->isOne() && SE.getSCEV(BO->getOperand(0)) == MaxRHS)
02068           NewRHS = BO->getOperand(0);
02069     if (!NewRHS)
02070       return Cond;
02071   } else if (SE.getSCEV(Sel->getOperand(1)) == MaxRHS)
02072     NewRHS = Sel->getOperand(1);
02073   else if (SE.getSCEV(Sel->getOperand(2)) == MaxRHS)
02074     NewRHS = Sel->getOperand(2);
02075   else if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(MaxRHS))
02076     NewRHS = SU->getValue();
02077   else
02078     // Max doesn't match expected pattern.
02079     return Cond;
02080 
02081   // Determine the new comparison opcode. It may be signed or unsigned,
02082   // and the original comparison may be either equality or inequality.
02083   if (Cond->getPredicate() == CmpInst::ICMP_EQ)
02084     Pred = CmpInst::getInversePredicate(Pred);
02085 
02086   // Ok, everything looks ok to change the condition into an SLT or SGE and
02087   // delete the max calculation.
02088   ICmpInst *NewCond =
02089     new ICmpInst(Cond, Pred, Cond->getOperand(0), NewRHS, "scmp");
02090 
02091   // Delete the max calculation instructions.
02092   Cond->replaceAllUsesWith(NewCond);
02093   CondUse->setUser(NewCond);
02094   Instruction *Cmp = cast<Instruction>(Sel->getOperand(0));
02095   Cond->eraseFromParent();
02096   Sel->eraseFromParent();
02097   if (Cmp->use_empty())
02098     Cmp->eraseFromParent();
02099   return NewCond;
02100 }
02101 
02102 /// OptimizeLoopTermCond - Change loop terminating condition to use the
02103 /// postinc iv when possible.
02104 void
02105 LSRInstance::OptimizeLoopTermCond() {
02106   SmallPtrSet<Instruction *, 4> PostIncs;
02107 
02108   BasicBlock *LatchBlock = L->getLoopLatch();
02109   SmallVector<BasicBlock*, 8> ExitingBlocks;
02110   L->getExitingBlocks(ExitingBlocks);
02111 
02112   for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) {
02113     BasicBlock *ExitingBlock = ExitingBlocks[i];
02114 
02115     // Get the terminating condition for the loop if possible.  If we
02116     // can, we want to change it to use a post-incremented version of its
02117     // induction variable, to allow coalescing the live ranges for the IV into
02118     // one register value.
02119 
02120     BranchInst *TermBr = dyn_cast<BranchInst>(ExitingBlock->getTerminator());
02121     if (!TermBr)
02122       continue;
02123     // FIXME: Overly conservative, termination condition could be an 'or' etc..
02124     if (TermBr->isUnconditional() || !isa<ICmpInst>(TermBr->getCondition()))
02125       continue;
02126 
02127     // Search IVUsesByStride to find Cond's IVUse if there is one.
02128     IVStrideUse *CondUse = nullptr;
02129     ICmpInst *Cond = cast<ICmpInst>(TermBr->getCondition());
02130     if (!FindIVUserForCond(Cond, CondUse))
02131       continue;
02132 
02133     // If the trip count is computed in terms of a max (due to ScalarEvolution
02134     // being unable to find a sufficient guard, for example), change the loop
02135     // comparison to use SLT or ULT instead of NE.
02136     // One consequence of doing this now is that it disrupts the count-down
02137     // optimization. That's not always a bad thing though, because in such
02138     // cases it may still be worthwhile to avoid a max.
02139     Cond = OptimizeMax(Cond, CondUse);
02140 
02141     // If this exiting block dominates the latch block, it may also use
02142     // the post-inc value if it won't be shared with other uses.
02143     // Check for dominance.
02144     if (!DT.dominates(ExitingBlock, LatchBlock))
02145       continue;
02146 
02147     // Conservatively avoid trying to use the post-inc value in non-latch
02148     // exits if there may be pre-inc users in intervening blocks.
02149     if (LatchBlock != ExitingBlock)
02150       for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI)
02151         // Test if the use is reachable from the exiting block. This dominator
02152         // query is a conservative approximation of reachability.
02153         if (&*UI != CondUse &&
02154             !DT.properlyDominates(UI->getUser()->getParent(), ExitingBlock)) {
02155           // Conservatively assume there may be reuse if the quotient of their
02156           // strides could be a legal scale.
02157           const SCEV *A = IU.getStride(*CondUse, L);
02158           const SCEV *B = IU.getStride(*UI, L);
02159           if (!A || !B) continue;
02160           if (SE.getTypeSizeInBits(A->getType()) !=
02161               SE.getTypeSizeInBits(B->getType())) {
02162             if (SE.getTypeSizeInBits(A->getType()) >
02163                 SE.getTypeSizeInBits(B->getType()))
02164               B = SE.getSignExtendExpr(B, A->getType());
02165             else
02166               A = SE.getSignExtendExpr(A, B->getType());
02167           }
02168           if (const SCEVConstant *D =
02169                 dyn_cast_or_null<SCEVConstant>(getExactSDiv(B, A, SE))) {
02170             const ConstantInt *C = D->getValue();
02171             // Stride of one or negative one can have reuse with non-addresses.
02172             if (C->isOne() || C->isAllOnesValue())
02173               goto decline_post_inc;
02174             // Avoid weird situations.
02175             if (C->getValue().getMinSignedBits() >= 64 ||
02176                 C->getValue().isMinSignedValue())
02177               goto decline_post_inc;
02178             // Check for possible scaled-address reuse.
02179             Type *AccessTy = getAccessType(UI->getUser());
02180             int64_t Scale = C->getSExtValue();
02181             if (TTI.isLegalAddressingMode(AccessTy, /*BaseGV=*/ nullptr,
02182                                           /*BaseOffset=*/ 0,
02183                                           /*HasBaseReg=*/ false, Scale))
02184               goto decline_post_inc;
02185             Scale = -Scale;
02186             if (TTI.isLegalAddressingMode(AccessTy, /*BaseGV=*/ nullptr,
02187                                           /*BaseOffset=*/ 0,
02188                                           /*HasBaseReg=*/ false, Scale))
02189               goto decline_post_inc;
02190           }
02191         }
02192 
02193     DEBUG(dbgs() << "  Change loop exiting icmp to use postinc iv: "
02194                  << *Cond << '\n');
02195 
02196     // It's possible for the setcc instruction to be anywhere in the loop, and
02197     // possible for it to have multiple users.  If it is not immediately before
02198     // the exiting block branch, move it.
02199     if (&*++BasicBlock::iterator(Cond) != TermBr) {
02200       if (Cond->hasOneUse()) {
02201         Cond->moveBefore(TermBr);
02202       } else {
02203         // Clone the terminating condition and insert into the loopend.
02204         ICmpInst *OldCond = Cond;
02205         Cond = cast<ICmpInst>(Cond->clone());
02206         Cond->setName(L->getHeader()->getName() + ".termcond");
02207         ExitingBlock->getInstList().insert(TermBr, Cond);
02208 
02209         // Clone the IVUse, as the old use still exists!
02210         CondUse = &IU.AddUser(Cond, CondUse->getOperandValToReplace());
02211         TermBr->replaceUsesOfWith(OldCond, Cond);
02212       }
02213     }
02214 
02215     // If we get to here, we know that we can transform the setcc instruction to
02216     // use the post-incremented version of the IV, allowing us to coalesce the
02217     // live ranges for the IV correctly.
02218     CondUse->transformToPostInc(L);
02219     Changed = true;
02220 
02221     PostIncs.insert(Cond);
02222   decline_post_inc:;
02223   }
02224 
02225   // Determine an insertion point for the loop induction variable increment. It
02226   // must dominate all the post-inc comparisons we just set up, and it must
02227   // dominate the loop latch edge.
02228   IVIncInsertPos = L->getLoopLatch()->getTerminator();
02229   for (SmallPtrSet<Instruction *, 4>::const_iterator I = PostIncs.begin(),
02230        E = PostIncs.end(); I != E; ++I) {
02231     BasicBlock *BB =
02232       DT.findNearestCommonDominator(IVIncInsertPos->getParent(),
02233                                     (*I)->getParent());
02234     if (BB == (*I)->getParent())
02235       IVIncInsertPos = *I;
02236     else if (BB != IVIncInsertPos->getParent())
02237       IVIncInsertPos = BB->getTerminator();
02238   }
02239 }
02240 
02241 /// reconcileNewOffset - Determine if the given use can accommodate a fixup
02242 /// at the given offset and other details. If so, update the use and
02243 /// return true.
02244 bool
02245 LSRInstance::reconcileNewOffset(LSRUse &LU, int64_t NewOffset, bool HasBaseReg,
02246                                 LSRUse::KindType Kind, Type *AccessTy) {
02247   int64_t NewMinOffset = LU.MinOffset;
02248   int64_t NewMaxOffset = LU.MaxOffset;
02249   Type *NewAccessTy = AccessTy;
02250 
02251   // Check for a mismatched kind. It's tempting to collapse mismatched kinds to
02252   // something conservative, however this can pessimize in the case that one of
02253   // the uses will have all its uses outside the loop, for example.
02254   if (LU.Kind != Kind)
02255     return false;
02256 
02257   // Check for a mismatched access type, and fall back conservatively as needed.
02258   // TODO: Be less conservative when the type is similar and can use the same
02259   // addressing modes.
02260   if (Kind == LSRUse::Address && AccessTy != LU.AccessTy)
02261     NewAccessTy = Type::getVoidTy(AccessTy->getContext());
02262 
02263   // Conservatively assume HasBaseReg is true for now.
02264   if (NewOffset < LU.MinOffset) {
02265     if (!isAlwaysFoldable(TTI, Kind, NewAccessTy, /*BaseGV=*/nullptr,
02266                           LU.MaxOffset - NewOffset, HasBaseReg))
02267       return false;
02268     NewMinOffset = NewOffset;
02269   } else if (NewOffset > LU.MaxOffset) {
02270     if (!isAlwaysFoldable(TTI, Kind, NewAccessTy, /*BaseGV=*/nullptr,
02271                           NewOffset - LU.MinOffset, HasBaseReg))
02272       return false;
02273     NewMaxOffset = NewOffset;
02274   }
02275 
02276   // Update the use.
02277   LU.MinOffset = NewMinOffset;
02278   LU.MaxOffset = NewMaxOffset;
02279   LU.AccessTy = NewAccessTy;
02280   if (NewOffset != LU.Offsets.back())
02281     LU.Offsets.push_back(NewOffset);
02282   return true;
02283 }
02284 
02285 /// getUse - Return an LSRUse index and an offset value for a fixup which
02286 /// needs the given expression, with the given kind and optional access type.
02287 /// Either reuse an existing use or create a new one, as needed.
02288 std::pair<size_t, int64_t>
02289 LSRInstance::getUse(const SCEV *&Expr,
02290                     LSRUse::KindType Kind, Type *AccessTy) {
02291   const SCEV *Copy = Expr;
02292   int64_t Offset = ExtractImmediate(Expr, SE);
02293 
02294   // Basic uses can't accept any offset, for example.
02295   if (!isAlwaysFoldable(TTI, Kind, AccessTy, /*BaseGV=*/ nullptr,
02296                         Offset, /*HasBaseReg=*/ true)) {
02297     Expr = Copy;
02298     Offset = 0;
02299   }
02300 
02301   std::pair<UseMapTy::iterator, bool> P =
02302     UseMap.insert(std::make_pair(LSRUse::SCEVUseKindPair(Expr, Kind), 0));
02303   if (!P.second) {
02304     // A use already existed with this base.
02305     size_t LUIdx = P.first->second;
02306     LSRUse &LU = Uses[LUIdx];
02307     if (reconcileNewOffset(LU, Offset, /*HasBaseReg=*/true, Kind, AccessTy))
02308       // Reuse this use.
02309       return std::make_pair(LUIdx, Offset);
02310   }
02311 
02312   // Create a new use.
02313   size_t LUIdx = Uses.size();
02314   P.first->second = LUIdx;
02315   Uses.push_back(LSRUse(Kind, AccessTy));
02316   LSRUse &LU = Uses[LUIdx];
02317 
02318   // We don't need to track redundant offsets, but we don't need to go out
02319   // of our way here to avoid them.
02320   if (LU.Offsets.empty() || Offset != LU.Offsets.back())
02321     LU.Offsets.push_back(Offset);
02322 
02323   LU.MinOffset = Offset;
02324   LU.MaxOffset = Offset;
02325   return std::make_pair(LUIdx, Offset);
02326 }
02327 
02328 /// DeleteUse - Delete the given use from the Uses list.
02329 void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) {
02330   if (&LU != &Uses.back())
02331     std::swap(LU, Uses.back());
02332   Uses.pop_back();
02333 
02334   // Update RegUses.
02335   RegUses.SwapAndDropUse(LUIdx, Uses.size());
02336 }
02337 
02338 /// FindUseWithFormula - Look for a use distinct from OrigLU which is has
02339 /// a formula that has the same registers as the given formula.
02340 LSRUse *
02341 LSRInstance::FindUseWithSimilarFormula(const Formula &OrigF,
02342                                        const LSRUse &OrigLU) {
02343   // Search all uses for the formula. This could be more clever.
02344   for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
02345     LSRUse &LU = Uses[LUIdx];
02346     // Check whether this use is close enough to OrigLU, to see whether it's
02347     // worthwhile looking through its formulae.
02348     // Ignore ICmpZero uses because they may contain formulae generated by
02349     // GenerateICmpZeroScales, in which case adding fixup offsets may
02350     // be invalid.
02351     if (&LU != &OrigLU &&
02352         LU.Kind != LSRUse::ICmpZero &&
02353         LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy &&
02354         LU.WidestFixupType == OrigLU.WidestFixupType &&
02355         LU.HasFormulaWithSameRegs(OrigF)) {
02356       // Scan through this use's formulae.
02357       for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(),
02358            E = LU.Formulae.end(); I != E; ++I) {
02359         const Formula &F = *I;
02360         // Check to see if this formula has the same registers and symbols
02361         // as OrigF.
02362         if (F.BaseRegs == OrigF.BaseRegs &&
02363             F.ScaledReg == OrigF.ScaledReg &&
02364             F.BaseGV == OrigF.BaseGV &&
02365             F.Scale == OrigF.Scale &&
02366             F.UnfoldedOffset == OrigF.UnfoldedOffset) {
02367           if (F.BaseOffset == 0)
02368             return &LU;
02369           // This is the formula where all the registers and symbols matched;
02370           // there aren't going to be any others. Since we declined it, we
02371           // can skip the rest of the formulae and proceed to the next LSRUse.
02372           break;
02373         }
02374       }
02375     }
02376   }
02377 
02378   // Nothing looked good.
02379   return nullptr;
02380 }
02381 
02382 void LSRInstance::CollectInterestingTypesAndFactors() {
02383   SmallSetVector<const SCEV *, 4> Strides;
02384 
02385   // Collect interesting types and strides.
02386   SmallVector<const SCEV *, 4> Worklist;
02387   for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) {
02388     const SCEV *Expr = IU.getExpr(*UI);
02389 
02390     // Collect interesting types.
02391     Types.insert(SE.getEffectiveSCEVType(Expr->getType()));
02392 
02393     // Add strides for mentioned loops.
02394     Worklist.push_back(Expr);
02395     do {
02396       const SCEV *S = Worklist.pop_back_val();
02397       if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
02398         if (AR->getLoop() == L)
02399           Strides.insert(AR->getStepRecurrence(SE));
02400         Worklist.push_back(AR->getStart());
02401       } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
02402         Worklist.append(Add->op_begin(), Add->op_end());
02403       }
02404     } while (!Worklist.empty());
02405   }
02406 
02407   // Compute interesting factors from the set of interesting strides.
02408   for (SmallSetVector<const SCEV *, 4>::const_iterator
02409        I = Strides.begin(), E = Strides.end(); I != E; ++I)
02410     for (SmallSetVector<const SCEV *, 4>::const_iterator NewStrideIter =
02411          std::next(I); NewStrideIter != E; ++NewStrideIter) {
02412       const SCEV *OldStride = *I;
02413       const SCEV *NewStride = *NewStrideIter;
02414 
02415       if (SE.getTypeSizeInBits(OldStride->getType()) !=
02416           SE.getTypeSizeInBits(NewStride->getType())) {
02417         if (SE.getTypeSizeInBits(OldStride->getType()) >
02418             SE.getTypeSizeInBits(NewStride->getType()))
02419           NewStride = SE.getSignExtendExpr(NewStride, OldStride->getType());
02420         else
02421           OldStride = SE.getSignExtendExpr(OldStride, NewStride->getType());
02422       }
02423       if (const SCEVConstant *Factor =
02424             dyn_cast_or_null<SCEVConstant>(getExactSDiv(NewStride, OldStride,
02425                                                         SE, true))) {
02426         if (Factor->getValue()->getValue().getMinSignedBits() <= 64)
02427           Factors.insert(Factor->getValue()->getValue().getSExtValue());
02428       } else if (const SCEVConstant *Factor =
02429                    dyn_cast_or_null<SCEVConstant>(getExactSDiv(OldStride,
02430                                                                NewStride,
02431                                                                SE, true))) {
02432         if (Factor->getValue()->getValue().getMinSignedBits() <= 64)
02433           Factors.insert(Factor->getValue()->getValue().getSExtValue());
02434       }
02435     }
02436 
02437   // If all uses use the same type, don't bother looking for truncation-based
02438   // reuse.
02439   if (Types.size() == 1)
02440     Types.clear();
02441 
02442   DEBUG(print_factors_and_types(dbgs()));
02443 }
02444 
02445 /// findIVOperand - Helper for CollectChains that finds an IV operand (computed
02446 /// by an AddRec in this loop) within [OI,OE) or returns OE. If IVUsers mapped
02447 /// Instructions to IVStrideUses, we could partially skip this.
02448 static User::op_iterator
02449 findIVOperand(User::op_iterator OI, User::op_iterator OE,
02450               Loop *L, ScalarEvolution &SE) {
02451   for(; OI != OE; ++OI) {
02452     if (Instruction *Oper = dyn_cast<Instruction>(*OI)) {
02453       if (!SE.isSCEVable(Oper->getType()))
02454         continue;
02455 
02456       if (const SCEVAddRecExpr *AR =
02457           dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Oper))) {
02458         if (AR->getLoop() == L)
02459           break;
02460       }
02461     }
02462   }
02463   return OI;
02464 }
02465 
02466 /// getWideOperand - IVChain logic must consistenctly peek base TruncInst
02467 /// operands, so wrap it in a convenient helper.
02468 static Value *getWideOperand(Value *Oper) {
02469   if (TruncInst *Trunc = dyn_cast<TruncInst>(Oper))
02470     return Trunc->getOperand(0);
02471   return Oper;
02472 }
02473 
02474 /// isCompatibleIVType - Return true if we allow an IV chain to include both
02475 /// types.
02476 static bool isCompatibleIVType(Value *LVal, Value *RVal) {
02477   Type *LType = LVal->getType();
02478   Type *RType = RVal->getType();
02479   return (LType == RType) || (LType->isPointerTy() && RType->isPointerTy());
02480 }
02481 
02482 /// getExprBase - Return an approximation of this SCEV expression's "base", or
02483 /// NULL for any constant. Returning the expression itself is
02484 /// conservative. Returning a deeper subexpression is more precise and valid as
02485 /// long as it isn't less complex than another subexpression. For expressions
02486 /// involving multiple unscaled values, we need to return the pointer-type
02487 /// SCEVUnknown. This avoids forming chains across objects, such as:
02488 /// PrevOper==a[i], IVOper==b[i], IVInc==b-a.
02489 ///
02490 /// Since SCEVUnknown is the rightmost type, and pointers are the rightmost
02491 /// SCEVUnknown, we simply return the rightmost SCEV operand.
02492 static const SCEV *getExprBase(const SCEV *S) {
02493   switch (S->getSCEVType()) {
02494   default: // uncluding scUnknown.
02495     return S;
02496   case scConstant:
02497     return nullptr;
02498   case scTruncate:
02499     return getExprBase(cast<SCEVTruncateExpr>(S)->getOperand());
02500   case scZeroExtend:
02501     return getExprBase(cast<SCEVZeroExtendExpr>(S)->getOperand());
02502   case scSignExtend:
02503     return getExprBase(cast<SCEVSignExtendExpr>(S)->getOperand());
02504   case scAddExpr: {
02505     // Skip over scaled operands (scMulExpr) to follow add operands as long as
02506     // there's nothing more complex.
02507     // FIXME: not sure if we want to recognize negation.
02508     const SCEVAddExpr *Add = cast<SCEVAddExpr>(S);
02509     for (std::reverse_iterator<SCEVAddExpr::op_iterator> I(Add->op_end()),
02510            E(Add->op_begin()); I != E; ++I) {
02511       const SCEV *SubExpr = *I;
02512       if (SubExpr->getSCEVType() == scAddExpr)
02513         return getExprBase(SubExpr);
02514 
02515       if (SubExpr->getSCEVType() != scMulExpr)
02516         return SubExpr;
02517     }
02518     return S; // all operands are scaled, be conservative.
02519   }
02520   case scAddRecExpr:
02521     return getExprBase(cast<SCEVAddRecExpr>(S)->getStart());
02522   }
02523 }
02524 
02525 /// Return true if the chain increment is profitable to expand into a loop
02526 /// invariant value, which may require its own register. A profitable chain
02527 /// increment will be an offset relative to the same base. We allow such offsets
02528 /// to potentially be used as chain increment as long as it's not obviously
02529 /// expensive to expand using real instructions.
02530 bool IVChain::isProfitableIncrement(const SCEV *OperExpr,
02531                                     const SCEV *IncExpr,
02532                                     ScalarEvolution &SE) {
02533   // Aggressively form chains when -stress-ivchain.
02534   if (StressIVChain)
02535     return true;
02536 
02537   // Do not replace a constant offset from IV head with a nonconstant IV
02538   // increment.
02539   if (!isa<SCEVConstant>(IncExpr)) {
02540     const SCEV *HeadExpr = SE.getSCEV(getWideOperand(Incs[0].IVOperand));
02541     if (isa<SCEVConstant>(SE.getMinusSCEV(OperExpr, HeadExpr)))
02542       return 0;
02543   }
02544 
02545   SmallPtrSet<const SCEV*, 8> Processed;
02546   return !isHighCostExpansion(IncExpr, Processed, SE);
02547 }
02548 
02549 /// Return true if the number of registers needed for the chain is estimated to
02550 /// be less than the number required for the individual IV users. First prohibit
02551 /// any IV users that keep the IV live across increments (the Users set should
02552 /// be empty). Next count the number and type of increments in the chain.
02553 ///
02554 /// Chaining IVs can lead to considerable code bloat if ISEL doesn't
02555 /// effectively use postinc addressing modes. Only consider it profitable it the
02556 /// increments can be computed in fewer registers when chained.
02557 ///
02558 /// TODO: Consider IVInc free if it's already used in another chains.
02559 static bool
02560 isProfitableChain(IVChain &Chain, SmallPtrSet<Instruction*, 4> &Users,
02561                   ScalarEvolution &SE, const TargetTransformInfo &TTI) {
02562   if (StressIVChain)
02563     return true;
02564 
02565   if (!Chain.hasIncs())
02566     return false;
02567 
02568   if (!Users.empty()) {
02569     DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " users:\n";
02570           for (SmallPtrSet<Instruction*, 4>::const_iterator I = Users.begin(),
02571                  E = Users.end(); I != E; ++I) {
02572             dbgs() << "  " << **I << "\n";
02573           });
02574     return false;
02575   }
02576   assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
02577 
02578   // The chain itself may require a register, so intialize cost to 1.
02579   int cost = 1;
02580 
02581   // A complete chain likely eliminates the need for keeping the original IV in
02582   // a register. LSR does not currently know how to form a complete chain unless
02583   // the header phi already exists.
02584   if (isa<PHINode>(Chain.tailUserInst())
02585       && SE.getSCEV(Chain.tailUserInst()) == Chain.Incs[0].IncExpr) {
02586     --cost;
02587   }
02588   const SCEV *LastIncExpr = nullptr;
02589   unsigned NumConstIncrements = 0;
02590   unsigned NumVarIncrements = 0;
02591   unsigned NumReusedIncrements = 0;
02592   for (IVChain::const_iterator I = Chain.begin(), E = Chain.end();
02593        I != E; ++I) {
02594 
02595     if (I->IncExpr->isZero())
02596       continue;
02597 
02598     // Incrementing by zero or some constant is neutral. We assume constants can
02599     // be folded into an addressing mode or an add's immediate operand.
02600     if (isa<SCEVConstant>(I->IncExpr)) {
02601       ++NumConstIncrements;
02602       continue;
02603     }
02604 
02605     if (I->IncExpr == LastIncExpr)
02606       ++NumReusedIncrements;
02607     else
02608       ++NumVarIncrements;
02609 
02610     LastIncExpr = I->IncExpr;
02611   }
02612   // An IV chain with a single increment is handled by LSR's postinc
02613   // uses. However, a chain with multiple increments requires keeping the IV's
02614   // value live longer than it needs to be if chained.
02615   if (NumConstIncrements > 1)
02616     --cost;
02617 
02618   // Materializing increment expressions in the preheader that didn't exist in
02619   // the original code may cost a register. For example, sign-extended array
02620   // indices can produce ridiculous increments like this:
02621   // IV + ((sext i32 (2 * %s) to i64) + (-1 * (sext i32 %s to i64)))
02622   cost += NumVarIncrements;
02623 
02624   // Reusing variable increments likely saves a register to hold the multiple of
02625   // the stride.
02626   cost -= NumReusedIncrements;
02627 
02628   DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " Cost: " << cost
02629                << "\n");
02630 
02631   return cost < 0;
02632 }
02633 
02634 /// ChainInstruction - Add this IV user to an existing chain or make it the head
02635 /// of a new chain.
02636 void LSRInstance::ChainInstruction(Instruction *UserInst, Instruction *IVOper,
02637                                    SmallVectorImpl<ChainUsers> &ChainUsersVec) {
02638   // When IVs are used as types of varying widths, they are generally converted
02639   // to a wider type with some uses remaining narrow under a (free) trunc.
02640   Value *const NextIV = getWideOperand(IVOper);
02641   const SCEV *const OperExpr = SE.getSCEV(NextIV);
02642   const SCEV *const OperExprBase = getExprBase(OperExpr);
02643 
02644   // Visit all existing chains. Check if its IVOper can be computed as a
02645   // profitable loop invariant increment from the last link in the Chain.
02646   unsigned ChainIdx = 0, NChains = IVChainVec.size();
02647   const SCEV *LastIncExpr = nullptr;
02648   for (; ChainIdx < NChains; ++ChainIdx) {
02649     IVChain &Chain = IVChainVec[ChainIdx];
02650 
02651     // Prune the solution space aggressively by checking that both IV operands
02652     // are expressions that operate on the same unscaled SCEVUnknown. This
02653     // "base" will be canceled by the subsequent getMinusSCEV call. Checking
02654     // first avoids creating extra SCEV expressions.
02655     if (!StressIVChain && Chain.ExprBase != OperExprBase)
02656       continue;
02657 
02658     Value *PrevIV = getWideOperand(Chain.Incs.back().IVOperand);
02659     if (!isCompatibleIVType(PrevIV, NextIV))
02660       continue;
02661 
02662     // A phi node terminates a chain.
02663     if (isa<PHINode>(UserInst) && isa<PHINode>(Chain.tailUserInst()))
02664       continue;
02665 
02666     // The increment must be loop-invariant so it can be kept in a register.
02667     const SCEV *PrevExpr = SE.getSCEV(PrevIV);
02668     const SCEV *IncExpr = SE.getMinusSCEV(OperExpr, PrevExpr);
02669     if (!SE.isLoopInvariant(IncExpr, L))
02670       continue;
02671 
02672     if (Chain.isProfitableIncrement(OperExpr, IncExpr, SE)) {
02673       LastIncExpr = IncExpr;
02674       break;
02675     }
02676   }
02677   // If we haven't found a chain, create a new one, unless we hit the max. Don't
02678   // bother for phi nodes, because they must be last in the chain.
02679   if (ChainIdx == NChains) {
02680     if (isa<PHINode>(UserInst))
02681       return;
02682     if (NChains >= MaxChains && !StressIVChain) {
02683       DEBUG(dbgs() << "IV Chain Limit\n");
02684       return;
02685     }
02686     LastIncExpr = OperExpr;
02687     // IVUsers may have skipped over sign/zero extensions. We don't currently
02688     // attempt to form chains involving extensions unless they can be hoisted
02689     // into this loop's AddRec.
02690     if (!isa<SCEVAddRecExpr>(LastIncExpr))
02691       return;
02692     ++NChains;
02693     IVChainVec.push_back(IVChain(IVInc(UserInst, IVOper, LastIncExpr),
02694                                  OperExprBase));
02695     ChainUsersVec.resize(NChains);
02696     DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Head: (" << *UserInst
02697                  << ") IV=" << *LastIncExpr << "\n");
02698   } else {
02699     DEBUG(dbgs() << "IV Chain#" << ChainIdx << "  Inc: (" << *UserInst
02700                  << ") IV+" << *LastIncExpr << "\n");
02701     // Add this IV user to the end of the chain.
02702     IVChainVec[ChainIdx].add(IVInc(UserInst, IVOper, LastIncExpr));
02703   }
02704   IVChain &Chain = IVChainVec[ChainIdx];
02705 
02706   SmallPtrSet<Instruction*,4> &NearUsers = ChainUsersVec[ChainIdx].NearUsers;
02707   // This chain's NearUsers become FarUsers.
02708   if (!LastIncExpr->isZero()) {
02709     ChainUsersVec[ChainIdx].FarUsers.insert(NearUsers.begin(),
02710                                             NearUsers.end());
02711     NearUsers.clear();
02712   }
02713 
02714   // All other uses of IVOperand become near uses of the chain.
02715   // We currently ignore intermediate values within SCEV expressions, assuming
02716   // they will eventually be used be the current chain, or can be computed
02717   // from one of the chain increments. To be more precise we could
02718   // transitively follow its user and only add leaf IV users to the set.
02719   for (User *U : IVOper->users()) {
02720     Instruction *OtherUse = dyn_cast<Instruction>(U);
02721     if (!OtherUse)
02722       continue;
02723     // Uses in the chain will no longer be uses if the chain is formed.
02724     // Include the head of the chain in this iteration (not Chain.begin()).
02725     IVChain::const_iterator IncIter = Chain.Incs.begin();
02726     IVChain::const_iterator IncEnd = Chain.Incs.end();
02727     for( ; IncIter != IncEnd; ++IncIter) {
02728       if (IncIter->UserInst == OtherUse)
02729         break;
02730     }
02731     if (IncIter != IncEnd)
02732       continue;
02733 
02734     if (SE.isSCEVable(OtherUse->getType())
02735         && !isa<SCEVUnknown>(SE.getSCEV(OtherUse))
02736         && IU.isIVUserOrOperand(OtherUse)) {
02737       continue;
02738     }
02739     NearUsers.insert(OtherUse);
02740   }
02741 
02742   // Since this user is part of the chain, it's no longer considered a use
02743   // of the chain.
02744   ChainUsersVec[ChainIdx].FarUsers.erase(UserInst);
02745 }
02746 
02747 /// CollectChains - Populate the vector of Chains.
02748 ///
02749 /// This decreases ILP at the architecture level. Targets with ample registers,
02750 /// multiple memory ports, and no register renaming probably don't want
02751 /// this. However, such targets should probably disable LSR altogether.
02752 ///
02753 /// The job of LSR is to make a reasonable choice of induction variables across
02754 /// the loop. Subsequent passes can easily "unchain" computation exposing more
02755 /// ILP *within the loop* if the target wants it.
02756 ///
02757 /// Finding the best IV chain is potentially a scheduling problem. Since LSR
02758 /// will not reorder memory operations, it will recognize this as a chain, but
02759 /// will generate redundant IV increments. Ideally this would be corrected later
02760 /// by a smart scheduler:
02761 ///        = A[i]
02762 ///        = A[i+x]
02763 /// A[i]   =
02764 /// A[i+x] =
02765 ///
02766 /// TODO: Walk the entire domtree within this loop, not just the path to the
02767 /// loop latch. This will discover chains on side paths, but requires
02768 /// maintaining multiple copies of the Chains state.
02769 void LSRInstance::CollectChains() {
02770   DEBUG(dbgs() << "Collecting IV Chains.\n");
02771   SmallVector<ChainUsers, 8> ChainUsersVec;
02772 
02773   SmallVector<BasicBlock *,8> LatchPath;
02774   BasicBlock *LoopHeader = L->getHeader();
02775   for (DomTreeNode *Rung = DT.getNode(L->getLoopLatch());
02776        Rung->getBlock() != LoopHeader; Rung = Rung->getIDom()) {
02777     LatchPath.push_back(Rung->getBlock());
02778   }
02779   LatchPath.push_back(LoopHeader);
02780 
02781   // Walk the instruction stream from the loop header to the loop latch.
02782   for (SmallVectorImpl<BasicBlock *>::reverse_iterator
02783          BBIter = LatchPath.rbegin(), BBEnd = LatchPath.rend();
02784        BBIter != BBEnd; ++BBIter) {
02785     for (BasicBlock::iterator I = (*BBIter)->begin(), E = (*BBIter)->end();
02786          I != E; ++I) {
02787       // Skip instructions that weren't seen by IVUsers analysis.
02788       if (isa<PHINode>(I) || !IU.isIVUserOrOperand(I))
02789         continue;
02790 
02791       // Ignore users that are part of a SCEV expression. This way we only
02792       // consider leaf IV Users. This effectively rediscovers a portion of
02793       // IVUsers analysis but in program order this time.
02794       if (SE.isSCEVable(I->getType()) && !isa<SCEVUnknown>(SE.getSCEV(I)))
02795         continue;
02796 
02797       // Remove this instruction from any NearUsers set it may be in.
02798       for (unsigned ChainIdx = 0, NChains = IVChainVec.size();
02799            ChainIdx < NChains; ++ChainIdx) {
02800         ChainUsersVec[ChainIdx].NearUsers.erase(I);
02801       }
02802       // Search for operands that can be chained.
02803       SmallPtrSet<Instruction*, 4> UniqueOperands;
02804       User::op_iterator IVOpEnd = I->op_end();
02805       User::op_iterator IVOpIter = findIVOperand(I->op_begin(), IVOpEnd, L, SE);
02806       while (IVOpIter != IVOpEnd) {
02807         Instruction *IVOpInst = cast<Instruction>(*IVOpIter);
02808         if (UniqueOperands.insert(IVOpInst))
02809           ChainInstruction(I, IVOpInst, ChainUsersVec);
02810         IVOpIter = findIVOperand(std::next(IVOpIter), IVOpEnd, L, SE);
02811       }
02812     } // Continue walking down the instructions.
02813   } // Continue walking down the domtree.
02814   // Visit phi backedges to determine if the chain can generate the IV postinc.
02815   for (BasicBlock::iterator I = L->getHeader()->begin();
02816        PHINode *PN = dyn_cast<PHINode>(I); ++I) {
02817     if (!SE.isSCEVable(PN->getType()))
02818       continue;
02819 
02820     Instruction *IncV =
02821       dyn_cast<Instruction>(PN->getIncomingValueForBlock(L->getLoopLatch()));
02822     if (IncV)
02823       ChainInstruction(PN, IncV, ChainUsersVec);
02824   }
02825   // Remove any unprofitable chains.
02826   unsigned ChainIdx = 0;
02827   for (unsigned UsersIdx = 0, NChains = IVChainVec.size();
02828        UsersIdx < NChains; ++UsersIdx) {
02829     if (!isProfitableChain(IVChainVec[UsersIdx],
02830                            ChainUsersVec[UsersIdx].FarUsers, SE, TTI))
02831       continue;
02832     // Preserve the chain at UsesIdx.
02833     if (ChainIdx != UsersIdx)
02834       IVChainVec[ChainIdx] = IVChainVec[UsersIdx];
02835     FinalizeChain(IVChainVec[ChainIdx]);
02836     ++ChainIdx;
02837   }
02838   IVChainVec.resize(ChainIdx);
02839 }
02840 
02841 void LSRInstance::FinalizeChain(IVChain &Chain) {
02842   assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
02843   DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[0].UserInst << "\n");
02844 
02845   for (IVChain::const_iterator I = Chain.begin(), E = Chain.end();
02846        I != E; ++I) {
02847     DEBUG(dbgs() << "        Inc: " << *I->UserInst << "\n");
02848     User::op_iterator UseI =
02849       std::find(I->UserInst->op_begin(), I->UserInst->op_end(), I->IVOperand);
02850     assert(UseI != I->UserInst->op_end() && "cannot find IV operand");
02851     IVIncSet.insert(UseI);
02852   }
02853 }
02854 
02855 /// Return true if the IVInc can be folded into an addressing mode.
02856 static bool canFoldIVIncExpr(const SCEV *IncExpr, Instruction *UserInst,
02857                              Value *Operand, const TargetTransformInfo &TTI) {
02858   const SCEVConstant *IncConst = dyn_cast<SCEVConstant>(IncExpr);
02859   if (!IncConst || !isAddressUse(UserInst, Operand))
02860     return false;
02861 
02862   if (IncConst->getValue()->getValue().getMinSignedBits() > 64)
02863     return false;
02864 
02865   int64_t IncOffset = IncConst->getValue()->getSExtValue();
02866   if (!isAlwaysFoldable(TTI, LSRUse::Address,
02867                         getAccessType(UserInst), /*BaseGV=*/ nullptr,
02868                         IncOffset, /*HaseBaseReg=*/ false))
02869     return false;
02870 
02871   return true;
02872 }
02873 
02874 /// GenerateIVChains - Generate an add or subtract for each IVInc in a chain to
02875 /// materialize the IV user's operand from the previous IV user's operand.
02876 void LSRInstance::GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter,
02877                                   SmallVectorImpl<WeakVH> &DeadInsts) {
02878   // Find the new IVOperand for the head of the chain. It may have been replaced
02879   // by LSR.
02880   const IVInc &Head = Chain.Incs[0];
02881   User::op_iterator IVOpEnd = Head.UserInst->op_end();
02882   // findIVOperand returns IVOpEnd if it can no longer find a valid IV user.
02883   User::op_iterator IVOpIter = findIVOperand(Head.UserInst->op_begin(),
02884                                              IVOpEnd, L, SE);
02885   Value *IVSrc = nullptr;
02886   while (IVOpIter != IVOpEnd) {
02887     IVSrc = getWideOperand(*IVOpIter);
02888 
02889     // If this operand computes the expression that the chain needs, we may use
02890     // it. (Check this after setting IVSrc which is used below.)
02891     //
02892     // Note that if Head.IncExpr is wider than IVSrc, then this phi is too
02893     // narrow for the chain, so we can no longer use it. We do allow using a
02894     // wider phi, assuming the LSR checked for free truncation. In that case we
02895     // should already have a truncate on this operand such that
02896     // getSCEV(IVSrc) == IncExpr.
02897     if (SE.getSCEV(*IVOpIter) == Head.IncExpr
02898         || SE.getSCEV(IVSrc) == Head.IncExpr) {
02899       break;
02900     }
02901     IVOpIter = findIVOperand(std::next(IVOpIter), IVOpEnd, L, SE);
02902   }
02903   if (IVOpIter == IVOpEnd) {
02904     // Gracefully give up on this chain.
02905     DEBUG(dbgs() << "Concealed chain head: " << *Head.UserInst << "\n");
02906     return;
02907   }
02908 
02909   DEBUG(dbgs() << "Generate chain at: " << *IVSrc << "\n");
02910   Type *IVTy = IVSrc->getType();
02911   Type *IntTy = SE.getEffectiveSCEVType(IVTy);
02912   const SCEV *LeftOverExpr = nullptr;
02913   for (IVChain::const_iterator IncI = Chain.begin(),
02914          IncE = Chain.end(); IncI != IncE; ++IncI) {
02915 
02916     Instruction *InsertPt = IncI->UserInst;
02917     if (isa<PHINode>(InsertPt))
02918       InsertPt = L->getLoopLatch()->getTerminator();
02919 
02920     // IVOper will replace the current IV User's operand. IVSrc is the IV
02921     // value currently held in a register.
02922     Value *IVOper = IVSrc;
02923     if (!IncI->IncExpr->isZero()) {
02924       // IncExpr was the result of subtraction of two narrow values, so must
02925       // be signed.
02926       const SCEV *IncExpr = SE.getNoopOrSignExtend(IncI->IncExpr, IntTy);
02927       LeftOverExpr = LeftOverExpr ?
02928         SE.getAddExpr(LeftOverExpr, IncExpr) : IncExpr;
02929     }
02930     if (LeftOverExpr && !LeftOverExpr->isZero()) {
02931       // Expand the IV increment.
02932       Rewriter.clearPostInc();
02933       Value *IncV = Rewriter.expandCodeFor(LeftOverExpr, IntTy, InsertPt);
02934       const SCEV *IVOperExpr = SE.getAddExpr(SE.getUnknown(IVSrc),
02935                                              SE.getUnknown(IncV));
02936       IVOper = Rewriter.expandCodeFor(IVOperExpr, IVTy, InsertPt);
02937 
02938       // If an IV increment can't be folded, use it as the next IV value.
02939       if (!canFoldIVIncExpr(LeftOverExpr, IncI->UserInst, IncI->IVOperand,
02940                             TTI)) {
02941         assert(IVTy == IVOper->getType() && "inconsistent IV increment type");
02942         IVSrc = IVOper;
02943         LeftOverExpr = nullptr;
02944       }
02945     }
02946     Type *OperTy = IncI->IVOperand->getType();
02947     if (IVTy != OperTy) {
02948       assert(SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) &&
02949              "cannot extend a chained IV");
02950       IRBuilder<> Builder(InsertPt);
02951       IVOper = Builder.CreateTruncOrBitCast(IVOper, OperTy, "lsr.chain");
02952     }
02953     IncI->UserInst->replaceUsesOfWith(IncI->IVOperand, IVOper);
02954     DeadInsts.push_back(IncI->IVOperand);
02955   }
02956   // If LSR created a new, wider phi, we may also replace its postinc. We only
02957   // do this if we also found a wide value for the head of the chain.
02958   if (isa<PHINode>(Chain.tailUserInst())) {
02959     for (BasicBlock::iterator I = L->getHeader()->begin();
02960          PHINode *Phi = dyn_cast<PHINode>(I); ++I) {
02961       if (!isCompatibleIVType(Phi, IVSrc))
02962         continue;
02963       Instruction *PostIncV = dyn_cast<Instruction>(
02964         Phi->getIncomingValueForBlock(L->getLoopLatch()));
02965       if (!PostIncV || (SE.getSCEV(PostIncV) != SE.getSCEV(IVSrc)))
02966         continue;
02967       Value *IVOper = IVSrc;
02968       Type *PostIncTy = PostIncV->getType();
02969       if (IVTy != PostIncTy) {
02970         assert(PostIncTy->isPointerTy() && "mixing int/ptr IV types");
02971         IRBuilder<> Builder(L->getLoopLatch()->getTerminator());
02972         Builder.SetCurrentDebugLocation(PostIncV->getDebugLoc());
02973         IVOper = Builder.CreatePointerCast(IVSrc, PostIncTy, "lsr.chain");
02974       }
02975       Phi->replaceUsesOfWith(PostIncV, IVOper);
02976       DeadInsts.push_back(PostIncV);
02977     }
02978   }
02979 }
02980 
02981 void LSRInstance::CollectFixupsAndInitialFormulae() {
02982   for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) {
02983     Instruction *UserInst = UI->getUser();
02984     // Skip IV users that are part of profitable IV Chains.
02985     User::op_iterator UseI = std::find(UserInst->op_begin(), UserInst->op_end(),
02986                                        UI->getOperandValToReplace());
02987     assert(UseI != UserInst->op_end() && "cannot find IV operand");
02988     if (IVIncSet.count(UseI))
02989       continue;
02990 
02991     // Record the uses.
02992     LSRFixup &LF = getNewFixup();
02993     LF.UserInst = UserInst;
02994     LF.OperandValToReplace = UI->getOperandValToReplace();
02995     LF.PostIncLoops = UI->getPostIncLoops();
02996 
02997     LSRUse::KindType Kind = LSRUse::Basic;
02998     Type *AccessTy = nullptr;
02999     if (isAddressUse(LF.UserInst, LF.OperandValToReplace)) {
03000       Kind = LSRUse::Address;
03001       AccessTy = getAccessType(LF.UserInst);
03002     }
03003 
03004     const SCEV *S = IU.getExpr(*UI);
03005 
03006     // Equality (== and !=) ICmps are special. We can rewrite (i == N) as
03007     // (N - i == 0), and this allows (N - i) to be the expression that we work
03008     // with rather than just N or i, so we can consider the register
03009     // requirements for both N and i at the same time. Limiting this code to
03010     // equality icmps is not a problem because all interesting loops use
03011     // equality icmps, thanks to IndVarSimplify.
03012     if (ICmpInst *CI = dyn_cast<ICmpInst>(LF.UserInst))
03013       if (CI->isEquality()) {
03014         // Swap the operands if needed to put the OperandValToReplace on the
03015         // left, for consistency.
03016         Value *NV = CI->getOperand(1);
03017         if (NV == LF.OperandValToReplace) {
03018           CI->setOperand(1, CI->getOperand(0));
03019           CI->setOperand(0, NV);
03020           NV = CI->getOperand(1);
03021           Changed = true;
03022         }
03023 
03024         // x == y  -->  x - y == 0
03025         const SCEV *N = SE.getSCEV(NV);
03026         if (SE.isLoopInvariant(N, L) && isSafeToExpand(N, SE)) {
03027           // S is normalized, so normalize N before folding it into S
03028           // to keep the result normalized.
03029           N = TransformForPostIncUse(Normalize, N, CI, nullptr,
03030                                      LF.PostIncLoops, SE, DT);
03031           Kind = LSRUse::ICmpZero;
03032           S = SE.getMinusSCEV(N, S);
03033         }
03034 
03035         // -1 and the negations of all interesting strides (except the negation
03036         // of -1) are now also interesting.
03037         for (size_t i = 0, e = Factors.size(); i != e; ++i)
03038           if (Factors[i] != -1)
03039             Factors.insert(-(uint64_t)Factors[i]);
03040         Factors.insert(-1);
03041       }
03042 
03043     // Set up the initial formula for this use.
03044     std::pair<size_t, int64_t> P = getUse(S, Kind, AccessTy);
03045     LF.LUIdx = P.first;
03046     LF.Offset = P.second;
03047     LSRUse &LU = Uses[LF.LUIdx];
03048     LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
03049     if (!LU.WidestFixupType ||
03050         SE.getTypeSizeInBits(LU.WidestFixupType) <
03051         SE.getTypeSizeInBits(LF.OperandValToReplace->getType()))
03052       LU.WidestFixupType = LF.OperandValToReplace->getType();
03053 
03054     // If this is the first use of this LSRUse, give it a formula.
03055     if (LU.Formulae.empty()) {
03056       InsertInitialFormula(S, LU, LF.LUIdx);
03057       CountRegisters(LU.Formulae.back(), LF.LUIdx);
03058     }
03059   }
03060 
03061   DEBUG(print_fixups(dbgs()));
03062 }
03063 
03064 /// InsertInitialFormula - Insert a formula for the given expression into
03065 /// the given use, separating out loop-variant portions from loop-invariant
03066 /// and loop-computable portions.
03067 void
03068 LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) {
03069   // Mark uses whose expressions cannot be expanded.
03070   if (!isSafeToExpand(S, SE))
03071     LU.RigidFormula = true;
03072 
03073   Formula F;
03074   F.InitialMatch(S, L, SE);
03075   bool Inserted = InsertFormula(LU, LUIdx, F);
03076   assert(Inserted && "Initial formula already exists!"); (void)Inserted;
03077 }
03078 
03079 /// InsertSupplementalFormula - Insert a simple single-register formula for
03080 /// the given expression into the given use.
03081 void
03082 LSRInstance::InsertSupplementalFormula(const SCEV *S,
03083                                        LSRUse &LU, size_t LUIdx) {
03084   Formula F;
03085   F.BaseRegs.push_back(S);
03086   F.HasBaseReg = true;
03087   bool Inserted = InsertFormula(LU, LUIdx, F);
03088   assert(Inserted && "Supplemental formula already exists!"); (void)Inserted;
03089 }
03090 
03091 /// CountRegisters - Note which registers are used by the given formula,
03092 /// updating RegUses.
03093 void LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) {
03094   if (F.ScaledReg)
03095     RegUses.CountRegister(F.ScaledReg, LUIdx);
03096   for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(),
03097        E = F.BaseRegs.end(); I != E; ++I)
03098     RegUses.CountRegister(*I, LUIdx);
03099 }
03100 
03101 /// InsertFormula - If the given formula has not yet been inserted, add it to
03102 /// the list, and return true. Return false otherwise.
03103 bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) {
03104   // Do not insert formula that we will not be able to expand.
03105   assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&
03106          "Formula is illegal");
03107   if (!LU.InsertFormula(F))
03108     return false;
03109 
03110   CountRegisters(F, LUIdx);
03111   return true;
03112 }
03113 
03114 /// CollectLoopInvariantFixupsAndFormulae - Check for other uses of
03115 /// loop-invariant values which we're tracking. These other uses will pin these
03116 /// values in registers, making them less profitable for elimination.
03117 /// TODO: This currently misses non-constant addrec step registers.
03118 /// TODO: Should this give more weight to users inside the loop?
03119 void
03120 LSRInstance::CollectLoopInvariantFixupsAndFormulae() {
03121   SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end());
03122   SmallPtrSet<const SCEV *, 8> Inserted;
03123 
03124   while (!Worklist.empty()) {
03125     const SCEV *S = Worklist.pop_back_val();
03126 
03127     if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(S))
03128       Worklist.append(N->op_begin(), N->op_end());
03129     else if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(S))
03130       Worklist.push_back(C->getOperand());
03131     else if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) {
03132       Worklist.push_back(D->getLHS());
03133       Worklist.push_back(D->getRHS());
03134     } else if (const SCEVUnknown *US = dyn_cast<SCEVUnknown>(S)) {
03135       if (!Inserted.insert(US)) continue;
03136       const Value *V = US->getValue();
03137       if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
03138         // Look for instructions defined outside the loop.
03139         if (L->contains(Inst)) continue;
03140       } else if (isa<UndefValue>(V))
03141         // Undef doesn't have a live range, so it doesn't matter.
03142         continue;
03143       for (const Use &U : V->uses()) {
03144         const Instruction *UserInst = dyn_cast<Instruction>(U.getUser());
03145         // Ignore non-instructions.
03146         if (!UserInst)
03147           continue;
03148         // Ignore instructions in other functions (as can happen with
03149         // Constants).
03150         if (UserInst->getParent()->getParent() != L->getHeader()->getParent())
03151           continue;
03152         // Ignore instructions not dominated by the loop.
03153         const BasicBlock *UseBB = !isa<PHINode>(UserInst) ?
03154           UserInst->getParent() :
03155           cast<PHINode>(UserInst)->getIncomingBlock(
03156             PHINode::getIncomingValueNumForOperand(U.getOperandNo()));
03157         if (!DT.dominates(L->getHeader(), UseBB))
03158           continue;
03159         // Ignore uses which are part of other SCEV expressions, to avoid
03160         // analyzing them multiple times.
03161         if (SE.isSCEVable(UserInst->getType())) {
03162           const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst));
03163           // If the user is a no-op, look through to its uses.
03164           if (!isa<SCEVUnknown>(UserS))
03165             continue;
03166           if (UserS == US) {
03167             Worklist.push_back(
03168               SE.getUnknown(const_cast<Instruction *>(UserInst)));
03169             continue;
03170           }
03171         }
03172         // Ignore icmp instructions which are already being analyzed.
03173         if (const ICmpInst *ICI = dyn_cast<ICmpInst>(UserInst)) {
03174           unsigned OtherIdx = !U.getOperandNo();
03175           Value *OtherOp = const_cast<Value *>(ICI->getOperand(OtherIdx));
03176           if (SE.hasComputableLoopEvolution(SE.getSCEV(OtherOp), L))
03177             continue;
03178         }
03179 
03180         LSRFixup &LF = getNewFixup();
03181         LF.UserInst = const_cast<Instruction *>(UserInst);
03182         LF.OperandValToReplace = U;
03183         std::pair<size_t, int64_t> P = getUse(S, LSRUse::Basic, nullptr);
03184         LF.LUIdx = P.first;
03185         LF.Offset = P.second;
03186         LSRUse &LU = Uses[LF.LUIdx];
03187         LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
03188         if (!LU.WidestFixupType ||
03189             SE.getTypeSizeInBits(LU.WidestFixupType) <
03190             SE.getTypeSizeInBits(LF.OperandValToReplace->getType()))
03191           LU.WidestFixupType = LF.OperandValToReplace->getType();
03192         InsertSupplementalFormula(US, LU, LF.LUIdx);
03193         CountRegisters(LU.Formulae.back(), Uses.size() - 1);
03194         break;
03195       }
03196     }
03197   }
03198 }
03199 
03200 /// CollectSubexprs - Split S into subexpressions which can be pulled out into
03201 /// separate registers. If C is non-null, multiply each subexpression by C.
03202 ///
03203 /// Return remainder expression after factoring the subexpressions captured by
03204 /// Ops. If Ops is complete, return NULL.
03205 static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
03206                                    SmallVectorImpl<const SCEV *> &Ops,
03207                                    const Loop *L,
03208                                    ScalarEvolution &SE,
03209                                    unsigned Depth = 0) {
03210   // Arbitrarily cap recursion to protect compile time.
03211   if (Depth >= 3)
03212     return S;
03213 
03214   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
03215     // Break out add operands.
03216     for (SCEVAddExpr::op_iterator I = Add->op_begin(), E = Add->op_end();
03217          I != E; ++I) {
03218       const SCEV *Remainder = CollectSubexprs(*I, C, Ops, L, SE, Depth+1);
03219       if (Remainder)
03220         Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder);
03221     }
03222     return nullptr;
03223   } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
03224     // Split a non-zero base out of an addrec.
03225     if (AR->getStart()->isZero())
03226       return S;
03227 
03228     const SCEV *Remainder = CollectSubexprs(AR->getStart(),
03229                                             C, Ops, L, SE, Depth+1);
03230     // Split the non-zero AddRec unless it is part of a nested recurrence that
03231     // does not pertain to this loop.
03232     if (Remainder && (AR->getLoop() == L || !isa<SCEVAddRecExpr>(Remainder))) {
03233       Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder);
03234       Remainder = nullptr;
03235     }
03236     if (Remainder != AR->getStart()) {
03237       if (!Remainder)
03238         Remainder = SE.getConstant(AR->getType(), 0);
03239       return SE.getAddRecExpr(Remainder,
03240                               AR->getStepRecurrence(SE),
03241                               AR->getLoop(),
03242                               //FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
03243                               SCEV::FlagAnyWrap);
03244     }
03245   } else if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
03246     // Break (C * (a + b + c)) into C*a + C*b + C*c.
03247     if (Mul->getNumOperands() != 2)
03248       return S;
03249     if (const SCEVConstant *Op0 =
03250         dyn_cast<SCEVConstant>(Mul->getOperand(0))) {
03251       C = C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0;
03252       const SCEV *Remainder =
03253         CollectSubexprs(Mul->getOperand(1), C, Ops, L, SE, Depth+1);
03254       if (Remainder)
03255         Ops.push_back(SE.getMulExpr(C, Remainder));
03256       return nullptr;
03257     }
03258   }
03259   return S;
03260 }
03261 
03262 /// \brief Helper function for LSRInstance::GenerateReassociations.
03263 void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
03264                                              const Formula &Base,
03265                                              unsigned Depth, size_t Idx,
03266                                              bool IsScaledReg) {
03267   const SCEV *BaseReg = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
03268   SmallVector<const SCEV *, 8> AddOps;
03269   const SCEV *Remainder = CollectSubexprs(BaseReg, nullptr, AddOps, L, SE);
03270   if (Remainder)
03271     AddOps.push_back(Remainder);
03272 
03273   if (AddOps.size() == 1)
03274     return;
03275 
03276   for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(),
03277                                                      JE = AddOps.end();
03278        J != JE; ++J) {
03279 
03280     // Loop-variant "unknown" values are uninteresting; we won't be able to
03281     // do anything meaningful with them.
03282     if (isa<SCEVUnknown>(*J) && !SE.isLoopInvariant(*J, L))
03283       continue;
03284 
03285     // Don't pull a constant into a register if the constant could be folded
03286     // into an immediate field.
03287     if (isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
03288                          LU.AccessTy, *J, Base.getNumRegs() > 1))
03289       continue;
03290 
03291     // Collect all operands except *J.
03292     SmallVector<const SCEV *, 8> InnerAddOps(
03293         ((const SmallVector<const SCEV *, 8> &)AddOps).begin(), J);
03294     InnerAddOps.append(std::next(J),
03295                        ((const SmallVector<const SCEV *, 8> &)AddOps).end());
03296 
03297     // Don't leave just a constant behind in a register if the constant could
03298     // be folded into an immediate field.
03299     if (InnerAddOps.size() == 1 &&
03300         isAlwaysFoldable(TTI, SE, LU.MinOffset, LU.MaxOffset, LU.Kind,
03301                          LU.AccessTy, InnerAddOps[0], Base.getNumRegs() > 1))
03302       continue;
03303 
03304     const SCEV *InnerSum = SE.getAddExpr(InnerAddOps);
03305     if (InnerSum->isZero())
03306       continue;
03307     Formula F = Base;
03308 
03309     // Add the remaining pieces of the add back into the new formula.
03310     const SCEVConstant *InnerSumSC = dyn_cast<SCEVConstant>(InnerSum);
03311     if (InnerSumSC && SE.getTypeSizeInBits(InnerSumSC->getType()) <= 64 &&
03312         TTI.isLegalAddImmediate((uint64_t)F.UnfoldedOffset +
03313                                 InnerSumSC->getValue()->getZExtValue())) {
03314       F.UnfoldedOffset =
03315           (uint64_t)F.UnfoldedOffset + InnerSumSC->getValue()->getZExtValue();
03316       if (IsScaledReg)
03317         F.ScaledReg = nullptr;
03318       else
03319         F.BaseRegs.erase(F.BaseRegs.begin() + Idx);
03320     } else if (IsScaledReg)
03321       F.ScaledReg = InnerSum;
03322     else
03323       F.BaseRegs[Idx] = InnerSum;
03324 
03325     // Add J as its own register, or an unfolded immediate.
03326     const SCEVConstant *SC = dyn_cast<SCEVConstant>(*J);
03327     if (SC && SE.getTypeSizeInBits(SC->getType()) <= 64 &&
03328         TTI.isLegalAddImmediate((uint64_t)F.UnfoldedOffset +
03329                                 SC->getValue()->getZExtValue()))
03330       F.UnfoldedOffset =
03331           (uint64_t)F.UnfoldedOffset + SC->getValue()->getZExtValue();
03332     else
03333       F.BaseRegs.push_back(*J);
03334     // We may have changed the number of register in base regs, adjust the
03335     // formula accordingly.
03336     F.Canonicalize();
03337 
03338     if (InsertFormula(LU, LUIdx, F))
03339       // If that formula hadn't been seen before, recurse to find more like
03340       // it.
03341       GenerateReassociations(LU, LUIdx, LU.Formulae.back(), Depth + 1);
03342   }
03343 }
03344 
03345 /// GenerateReassociations - Split out subexpressions from adds and the bases of
03346 /// addrecs.
03347 void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx,
03348                                          Formula Base, unsigned Depth) {
03349   assert(Base.isCanonical() && "Input must be in the canonical form");
03350   // Arbitrarily cap recursion to protect compile time.
03351   if (Depth >= 3)
03352     return;
03353 
03354   for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
03355     GenerateReassociationsImpl(LU, LUIdx, Base, Depth, i);
03356 
03357   if (Base.Scale == 1)
03358     GenerateReassociationsImpl(LU, LUIdx, Base, Depth,
03359                                /* Idx */ -1, /* IsScaledReg */ true);
03360 }
03361 
03362 /// GenerateCombinations - Generate a formula consisting of all of the
03363 /// loop-dominating registers added into a single register.
03364 void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
03365                                        Formula Base) {
03366   // This method is only interesting on a plurality of registers.
03367   if (Base.BaseRegs.size() + (Base.Scale == 1) <= 1)
03368     return;
03369 
03370   // Flatten the representation, i.e., reg1 + 1*reg2 => reg1 + reg2, before
03371   // processing the formula.
03372   Base.Unscale();
03373   Formula F = Base;
03374   F.BaseRegs.clear();
03375   SmallVector<const SCEV *, 4> Ops;
03376   for (SmallVectorImpl<const SCEV *>::const_iterator
03377        I = Base.BaseRegs.begin(), E = Base.BaseRegs.end(); I != E; ++I) {
03378     const SCEV *BaseReg = *I;
03379     if (SE.properlyDominates(BaseReg, L->getHeader()) &&
03380         !SE.hasComputableLoopEvolution(BaseReg, L))
03381       Ops.push_back(BaseReg);
03382     else
03383       F.BaseRegs.push_back(BaseReg);
03384   }
03385   if (Ops.size() > 1) {
03386     const SCEV *Sum = SE.getAddExpr(Ops);
03387     // TODO: If Sum is zero, it probably means ScalarEvolution missed an
03388     // opportunity to fold something. For now, just ignore such cases
03389     // rather than proceed with zero in a register.
03390     if (!Sum->isZero()) {
03391       F.BaseRegs.push_back(Sum);
03392       F.Canonicalize();
03393       (void)InsertFormula(LU, LUIdx, F);
03394     }
03395   }
03396 }
03397 
03398 /// \brief Helper function for LSRInstance::GenerateSymbolicOffsets.
03399 void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
03400                                               const Formula &Base, size_t Idx,
03401                                               bool IsScaledReg) {
03402   const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
03403   GlobalValue *GV = ExtractSymbol(G, SE);
03404   if (G->isZero() || !GV)
03405     return;
03406   Formula F = Base;
03407   F.BaseGV = GV;
03408   if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
03409     return;
03410   if (IsScaledReg)
03411     F.ScaledReg = G;
03412   else
03413     F.BaseRegs[Idx] = G;
03414   (void)InsertFormula(LU, LUIdx, F);
03415 }
03416 
03417 /// GenerateSymbolicOffsets - Generate reuse formulae using symbolic offsets.
03418 void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
03419                                           Formula Base) {
03420   // We can't add a symbolic offset if the address already contains one.
03421   if (Base.BaseGV) return;
03422 
03423   for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
03424     GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, i);
03425   if (Base.Scale == 1)
03426     GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1,
03427                                 /* IsScaledReg */ true);
03428 }
03429 
03430 /// \brief Helper function for LSRInstance::GenerateConstantOffsets.
03431 void LSRInstance::GenerateConstantOffsetsImpl(
03432     LSRUse &LU, unsigned LUIdx, const Formula &Base,
03433     const SmallVectorImpl<int64_t> &Worklist, size_t Idx, bool IsScaledReg) {
03434   const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
03435   for (SmallVectorImpl<int64_t>::const_iterator I = Worklist.begin(),
03436                                                 E = Worklist.end();
03437        I != E; ++I) {
03438     Formula F = Base;
03439     F.BaseOffset = (uint64_t)Base.BaseOffset - *I;
03440     if (isLegalUse(TTI, LU.MinOffset - *I, LU.MaxOffset - *I, LU.Kind,
03441                    LU.AccessTy, F)) {
03442       // Add the offset to the base register.
03443       const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), *I), G);
03444       // If it cancelled out, drop the base register, otherwise update it.
03445       if (NewG->isZero()) {
03446         if (IsScaledReg) {
03447           F.Scale = 0;
03448           F.ScaledReg = nullptr;
03449         } else
03450           F.DeleteBaseReg(F.BaseRegs[Idx]);
03451         F.Canonicalize();
03452       } else if (IsScaledReg)
03453         F.ScaledReg = NewG;
03454       else
03455         F.BaseRegs[Idx] = NewG;
03456 
03457       (void)InsertFormula(LU, LUIdx, F);
03458     }
03459   }
03460 
03461   int64_t Imm = ExtractImmediate(G, SE);
03462   if (G->isZero() || Imm == 0)
03463     return;
03464   Formula F = Base;
03465   F.BaseOffset = (uint64_t)F.BaseOffset + Imm;
03466   if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F))
03467     return;
03468   if (IsScaledReg)
03469     F.ScaledReg = G;
03470   else
03471     F.BaseRegs[Idx] = G;
03472   (void)InsertFormula(LU, LUIdx, F);
03473 }
03474 
03475 /// GenerateConstantOffsets - Generate reuse formulae using symbolic offsets.
03476 void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx,
03477                                           Formula Base) {
03478   // TODO: For now, just add the min and max offset, because it usually isn't
03479   // worthwhile looking at everything inbetween.
03480   SmallVector<int64_t, 2> Worklist;
03481   Worklist.push_back(LU.MinOffset);
03482   if (LU.MaxOffset != LU.MinOffset)
03483     Worklist.push_back(LU.MaxOffset);
03484 
03485   for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
03486     GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, i);
03487   if (Base.Scale == 1)
03488     GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1,
03489                                 /* IsScaledReg */ true);
03490 }
03491 
03492 /// GenerateICmpZeroScales - For ICmpZero, check to see if we can scale up
03493 /// the comparison. For example, x == y -> x*c == y*c.
03494 void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
03495                                          Formula Base) {
03496   if (LU.Kind != LSRUse::ICmpZero) return;
03497 
03498   // Determine the integer type for the base formula.
03499   Type *IntTy = Base.getType();
03500   if (!IntTy) return;
03501   if (SE.getTypeSizeInBits(IntTy) > 64) return;
03502 
03503   // Don't do this if there is more than one offset.
03504   if (LU.MinOffset != LU.MaxOffset) return;
03505 
03506   assert(!Base.BaseGV && "ICmpZero use is not legal!");
03507 
03508   // Check each interesting stride.
03509   for (SmallSetVector<int64_t, 8>::const_iterator
03510        I = Factors.begin(), E = Factors.end(); I != E; ++I) {
03511     int64_t Factor = *I;
03512 
03513     // Check that the multiplication doesn't overflow.
03514     if (Base.BaseOffset == INT64_MIN && Factor == -1)
03515       continue;
03516     int64_t NewBaseOffset = (uint64_t)Base.BaseOffset * Factor;
03517     if (NewBaseOffset / Factor != Base.BaseOffset)
03518       continue;
03519     // If the offset will be truncated at this use, check that it is in bounds.
03520     if (!IntTy->isPointerTy() &&
03521         !ConstantInt::isValueValidForType(IntTy, NewBaseOffset))
03522       continue;
03523 
03524     // Check that multiplying with the use offset doesn't overflow.
03525     int64_t Offset = LU.MinOffset;
03526     if (Offset == INT64_MIN && Factor == -1)
03527       continue;
03528     Offset = (uint64_t)Offset * Factor;
03529     if (Offset / Factor != LU.MinOffset)
03530       continue;
03531     // If the offset will be truncated at this use, check that it is in bounds.
03532     if (!IntTy->isPointerTy() &&
03533         !ConstantInt::isValueValidForType(IntTy, Offset))
03534       continue;
03535 
03536     Formula F = Base;
03537     F.BaseOffset = NewBaseOffset;
03538 
03539     // Check that this scale is legal.
03540     if (!isLegalUse(TTI, Offset, Offset, LU.Kind, LU.AccessTy, F))
03541       continue;
03542 
03543     // Compensate for the use having MinOffset built into it.
03544     F.BaseOffset = (uint64_t)F.BaseOffset + Offset - LU.MinOffset;
03545 
03546     const SCEV *FactorS = SE.getConstant(IntTy, Factor);
03547 
03548     // Check that multiplying with each base register doesn't overflow.
03549     for (size_t i = 0, e = F.BaseRegs.size(); i != e; ++i) {
03550       F.BaseRegs[i] = SE.getMulExpr(F.BaseRegs[i], FactorS);
03551       if (getExactSDiv(F.BaseRegs[i], FactorS, SE) != Base.BaseRegs[i])
03552         goto next;
03553     }
03554 
03555     // Check that multiplying with the scaled register doesn't overflow.
03556     if (F.ScaledReg) {
03557       F.ScaledReg = SE.getMulExpr(F.ScaledReg, FactorS);
03558       if (getExactSDiv(F.ScaledReg, FactorS, SE) != Base.ScaledReg)
03559         continue;
03560     }
03561 
03562     // Check that multiplying with the unfolded offset doesn't overflow.
03563     if (F.UnfoldedOffset != 0) {
03564       if (F.UnfoldedOffset == INT64_MIN && Factor == -1)
03565         continue;
03566       F.UnfoldedOffset = (uint64_t)F.UnfoldedOffset * Factor;
03567       if (F.UnfoldedOffset / Factor != Base.UnfoldedOffset)
03568         continue;
03569       // If the offset will be truncated, check that it is in bounds.
03570       if (!IntTy->isPointerTy() &&
03571           !ConstantInt::isValueValidForType(IntTy, F.UnfoldedOffset))
03572         continue;
03573     }
03574 
03575     // If we make it here and it's legal, add it.
03576     (void)InsertFormula(LU, LUIdx, F);
03577   next:;
03578   }
03579 }
03580 
03581 /// GenerateScales - Generate stride factor reuse formulae by making use of
03582 /// scaled-offset address modes, for example.
03583 void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
03584   // Determine the integer type for the base formula.
03585   Type *IntTy = Base.getType();
03586   if (!IntTy) return;
03587 
03588   // If this Formula already has a scaled register, we can't add another one.
03589   // Try to unscale the formula to generate a better scale.
03590   if (Base.Scale != 0 && !Base.Unscale())
03591     return;
03592 
03593   assert(Base.Scale == 0 && "Unscale did not did its job!");
03594 
03595   // Check each interesting stride.
03596   for (SmallSetVector<int64_t, 8>::const_iterator
03597        I = Factors.begin(), E = Factors.end(); I != E; ++I) {
03598     int64_t Factor = *I;
03599 
03600     Base.Scale = Factor;
03601     Base.HasBaseReg = Base.BaseRegs.size() > 1;
03602     // Check whether this scale is going to be legal.
03603     if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
03604                     Base)) {
03605       // As a special-case, handle special out-of-loop Basic users specially.
03606       // TODO: Reconsider this special case.
03607       if (LU.Kind == LSRUse::Basic &&
03608           isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LSRUse::Special,
03609                      LU.AccessTy, Base) &&
03610           LU.AllFixupsOutsideLoop)
03611         LU.Kind = LSRUse::Special;
03612       else
03613         continue;
03614     }
03615     // For an ICmpZero, negating a solitary base register won't lead to
03616     // new solutions.
03617     if (LU.Kind == LSRUse::ICmpZero &&
03618         !Base.HasBaseReg && Base.BaseOffset == 0 && !Base.BaseGV)
03619       continue;
03620     // For each addrec base reg, apply the scale, if possible.
03621     for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
03622       if (const SCEVAddRecExpr *AR =
03623             dyn_cast<SCEVAddRecExpr>(Base.BaseRegs[i])) {
03624         const SCEV *FactorS = SE.getConstant(IntTy, Factor);
03625         if (FactorS->isZero())
03626           continue;
03627         // Divide out the factor, ignoring high bits, since we'll be
03628         // scaling the value back up in the end.
03629         if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true)) {
03630           // TODO: This could be optimized to avoid all the copying.
03631           Formula F = Base;
03632           F.ScaledReg = Quotient;
03633           F.DeleteBaseReg(F.BaseRegs[i]);
03634           // The canonical representation of 1*reg is reg, which is already in
03635           // Base. In that case, do not try to insert the formula, it will be
03636           // rejected anyway.
03637           if (F.Scale == 1 && F.BaseRegs.empty())
03638             continue;
03639           (void)InsertFormula(LU, LUIdx, F);
03640         }
03641       }
03642   }
03643 }
03644 
03645 /// GenerateTruncates - Generate reuse formulae from different IV types.
03646 void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
03647   // Don't bother truncating symbolic values.
03648   if (Base.BaseGV) return;
03649 
03650   // Determine the integer type for the base formula.
03651   Type *DstTy = Base.getType();
03652   if (!DstTy) return;
03653   DstTy = SE.getEffectiveSCEVType(DstTy);
03654 
03655   for (SmallSetVector<Type *, 4>::const_iterator
03656        I = Types.begin(), E = Types.end(); I != E; ++I) {
03657     Type *SrcTy = *I;
03658     if (SrcTy != DstTy && TTI.isTruncateFree(SrcTy, DstTy)) {
03659       Formula F = Base;
03660 
03661       if (F.ScaledReg) F.ScaledReg = SE.getAnyExtendExpr(F.ScaledReg, *I);
03662       for (SmallVectorImpl<const SCEV *>::iterator J = F.BaseRegs.begin(),
03663            JE = F.BaseRegs.end(); J != JE; ++J)
03664         *J = SE.getAnyExtendExpr(*J, SrcTy);
03665 
03666       // TODO: This assumes we've done basic processing on all uses and
03667       // have an idea what the register usage is.
03668       if (!F.hasRegsUsedByUsesOtherThan(LUIdx, RegUses))
03669         continue;
03670 
03671       (void)InsertFormula(LU, LUIdx, F);
03672     }
03673   }
03674 }
03675 
03676 namespace {
03677 
03678 /// WorkItem - Helper class for GenerateCrossUseConstantOffsets. It's used to
03679 /// defer modifications so that the search phase doesn't have to worry about
03680 /// the data structures moving underneath it.
03681 struct WorkItem {
03682   size_t LUIdx;
03683   int64_t Imm;
03684   const SCEV *OrigReg;
03685 
03686   WorkItem(size_t LI, int64_t I, const SCEV *R)
03687     : LUIdx(LI), Imm(I), OrigReg(R) {}
03688 
03689   void print(raw_ostream &OS) const;
03690   void dump() const;
03691 };
03692 
03693 }
03694 
03695 void WorkItem::print(raw_ostream &OS) const {
03696   OS << "in formulae referencing " << *OrigReg << " in use " << LUIdx
03697      << " , add offset " << Imm;
03698 }
03699 
03700 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
03701 void WorkItem::dump() const {
03702   print(errs()); errs() << '\n';
03703 }
03704 #endif
03705 
03706 /// GenerateCrossUseConstantOffsets - Look for registers which are a constant
03707 /// distance apart and try to form reuse opportunities between them.
03708 void LSRInstance::GenerateCrossUseConstantOffsets() {
03709   // Group the registers by their value without any added constant offset.
03710   typedef std::map<int64_t, const SCEV *> ImmMapTy;
03711   typedef DenseMap<const SCEV *, ImmMapTy> RegMapTy;
03712   RegMapTy Map;
03713   DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap;
03714   SmallVector<const SCEV *, 8> Sequence;
03715   for (RegUseTracker::const_iterator I = RegUses.begin(), E = RegUses.end();
03716        I != E; ++I) {
03717     const SCEV *Reg = *I;
03718     int64_t Imm = ExtractImmediate(Reg, SE);
03719     std::pair<RegMapTy::iterator, bool> Pair =
03720       Map.insert(std::make_pair(Reg, ImmMapTy()));
03721     if (Pair.second)
03722       Sequence.push_back(Reg);
03723     Pair.first->second.insert(std::make_pair(Imm, *I));
03724     UsedByIndicesMap[Reg] |= RegUses.getUsedByIndices(*I);
03725   }
03726 
03727   // Now examine each set of registers with the same base value. Build up
03728   // a list of work to do and do the work in a separate step so that we're
03729   // not adding formulae and register counts while we're searching.
03730   SmallVector<WorkItem, 32> WorkItems;
03731   SmallSet<std::pair<size_t, int64_t>, 32> UniqueItems;
03732   for (SmallVectorImpl<const SCEV *>::const_iterator I = Sequence.begin(),
03733        E = Sequence.end(); I != E; ++I) {
03734     const SCEV *Reg = *I;
03735     const ImmMapTy &Imms = Map.find(Reg)->second;
03736 
03737     // It's not worthwhile looking for reuse if there's only one offset.
03738     if (Imms.size() == 1)
03739       continue;
03740 
03741     DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << ':';
03742           for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end();
03743                J != JE; ++J)
03744             dbgs() << ' ' << J->first;
03745           dbgs() << '\n');
03746 
03747     // Examine each offset.
03748     for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end();
03749          J != JE; ++J) {
03750       const SCEV *OrigReg = J->second;
03751 
03752       int64_t JImm = J->first;
03753       const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(OrigReg);
03754 
03755       if (!isa<SCEVConstant>(OrigReg) &&
03756           UsedByIndicesMap[Reg].count() == 1) {
03757         DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg << '\n');
03758         continue;
03759       }
03760 
03761       // Conservatively examine offsets between this orig reg a few selected
03762       // other orig regs.
03763       ImmMapTy::const_iterator OtherImms[] = {
03764         Imms.begin(), std::prev(Imms.end()),
03765         Imms.lower_bound((Imms.begin()->first + std::prev(Imms.end())->first) /
03766                          2)
03767       };
03768       for (size_t i = 0, e = array_lengthof(OtherImms); i != e; ++i) {
03769         ImmMapTy::const_iterator M = OtherImms[i];
03770         if (M == J || M == JE) continue;
03771 
03772         // Compute the difference between the two.
03773         int64_t Imm = (uint64_t)JImm - M->first;
03774         for (int LUIdx = UsedByIndices.find_first(); LUIdx != -1;
03775              LUIdx = UsedByIndices.find_next(LUIdx))
03776           // Make a memo of this use, offset, and register tuple.
03777           if (UniqueItems.insert(std::make_pair(LUIdx, Imm)))
03778             WorkItems.push_back(WorkItem(LUIdx, Imm, OrigReg));
03779       }
03780     }
03781   }
03782 
03783   Map.clear();
03784   Sequence.clear();
03785   UsedByIndicesMap.clear();
03786   UniqueItems.clear();
03787 
03788   // Now iterate through the worklist and add new formulae.
03789   for (SmallVectorImpl<WorkItem>::const_iterator I = WorkItems.begin(),
03790        E = WorkItems.end(); I != E; ++I) {
03791     const WorkItem &WI = *I;
03792     size_t LUIdx = WI.LUIdx;
03793     LSRUse &LU = Uses[LUIdx];
03794     int64_t Imm = WI.Imm;
03795     const SCEV *OrigReg = WI.OrigReg;
03796 
03797     Type *IntTy = SE.getEffectiveSCEVType(OrigReg->getType());
03798     const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm));
03799     unsigned BitWidth = SE.getTypeSizeInBits(IntTy);
03800 
03801     // TODO: Use a more targeted data structure.
03802     for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) {
03803       Formula F = LU.Formulae[L];
03804       // FIXME: The code for the scaled and unscaled registers looks
03805       // very similar but slightly different. Investigate if they
03806       // could be merged. That way, we would not have to unscale the
03807       // Formula.
03808       F.Unscale();
03809       // Use the immediate in the scaled register.
03810       if (F.ScaledReg == OrigReg) {
03811         int64_t Offset = (uint64_t)F.BaseOffset + Imm * (uint64_t)F.Scale;
03812         // Don't create 50 + reg(-50).
03813         if (F.referencesReg(SE.getSCEV(
03814                    ConstantInt::get(IntTy, -(uint64_t)Offset))))
03815           continue;
03816         Formula NewF = F;
03817         NewF.BaseOffset = Offset;
03818         if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
03819                         NewF))
03820           continue;
03821         NewF.ScaledReg = SE.getAddExpr(NegImmS, NewF.ScaledReg);
03822 
03823         // If the new scale is a constant in a register, and adding the constant
03824         // value to the immediate would produce a value closer to zero than the
03825         // immediate itself, then the formula isn't worthwhile.
03826         if (const SCEVConstant *C = dyn_cast<SCEVConstant>(NewF.ScaledReg))
03827           if (C->getValue()->isNegative() !=
03828                 (NewF.BaseOffset < 0) &&
03829               (C->getValue()->getValue().abs() * APInt(BitWidth, F.Scale))
03830                 .ule(abs64(NewF.BaseOffset)))
03831             continue;
03832 
03833         // OK, looks good.
03834         NewF.Canonicalize();
03835         (void)InsertFormula(LU, LUIdx, NewF);
03836       } else {
03837         // Use the immediate in a base register.
03838         for (size_t N = 0, NE = F.BaseRegs.size(); N != NE; ++N) {
03839           const SCEV *BaseReg = F.BaseRegs[N];
03840           if (BaseReg != OrigReg)
03841             continue;
03842           Formula NewF = F;
03843           NewF.BaseOffset = (uint64_t)NewF.BaseOffset + Imm;
03844           if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset,
03845                           LU.Kind, LU.AccessTy, NewF)) {
03846             if (!TTI.isLegalAddImmediate((uint64_t)NewF.UnfoldedOffset + Imm))
03847               continue;
03848             NewF = F;
03849             NewF.UnfoldedOffset = (uint64_t)NewF.UnfoldedOffset + Imm;
03850           }
03851           NewF.BaseRegs[N] = SE.getAddExpr(NegImmS, BaseReg);
03852 
03853           // If the new formula has a constant in a register, and adding the
03854           // constant value to the immediate would produce a value closer to
03855           // zero than the immediate itself, then the formula isn't worthwhile.
03856           for (SmallVectorImpl<const SCEV *>::const_iterator
03857                J = NewF.BaseRegs.begin(), JE = NewF.BaseRegs.end();
03858                J != JE; ++J)
03859             if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*J))
03860               if ((C->getValue()->getValue() + NewF.BaseOffset).abs().slt(
03861                    abs64(NewF.BaseOffset)) &&
03862                   (C->getValue()->getValue() +
03863                    NewF.BaseOffset).countTrailingZeros() >=
03864                    countTrailingZeros<uint64_t>(NewF.BaseOffset))
03865                 goto skip_formula;
03866 
03867           // Ok, looks good.
03868           NewF.Canonicalize();
03869           (void)InsertFormula(LU, LUIdx, NewF);
03870           break;
03871         skip_formula:;
03872         }
03873       }
03874     }
03875   }
03876 }
03877 
03878 /// GenerateAllReuseFormulae - Generate formulae for each use.
03879 void
03880 LSRInstance::GenerateAllReuseFormulae() {
03881   // This is split into multiple loops so that hasRegsUsedByUsesOtherThan
03882   // queries are more precise.
03883   for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
03884     LSRUse &LU = Uses[LUIdx];
03885     for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
03886       GenerateReassociations(LU, LUIdx, LU.Formulae[i]);
03887     for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
03888       GenerateCombinations(LU, LUIdx, LU.Formulae[i]);
03889   }
03890   for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
03891     LSRUse &LU = Uses[LUIdx];
03892     for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
03893       GenerateSymbolicOffsets(LU, LUIdx, LU.Formulae[i]);
03894     for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
03895       GenerateConstantOffsets(LU, LUIdx, LU.Formulae[i]);
03896     for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
03897       GenerateICmpZeroScales(LU, LUIdx, LU.Formulae[i]);
03898     for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
03899       GenerateScales(LU, LUIdx, LU.Formulae[i]);
03900   }
03901   for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
03902     LSRUse &LU = Uses[LUIdx];
03903     for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
03904       GenerateTruncates(LU, LUIdx, LU.Formulae[i]);
03905   }
03906 
03907   GenerateCrossUseConstantOffsets();
03908 
03909   DEBUG(dbgs() << "\n"
03910                   "After generating reuse formulae:\n";
03911         print_uses(dbgs()));
03912 }
03913 
03914 /// If there are multiple formulae with the same set of registers used
03915 /// by other uses, pick the best one and delete the others.
03916 void LSRInstance::FilterOutUndesirableDedicatedRegisters() {
03917   DenseSet<const SCEV *> VisitedRegs;
03918   SmallPtrSet<const SCEV *, 16> Regs;
03919   SmallPtrSet<const SCEV *, 16> LoserRegs;
03920 #ifndef NDEBUG
03921   bool ChangedFormulae = false;
03922 #endif
03923 
03924   // Collect the best formula for each unique set of shared registers. This
03925   // is reset for each use.
03926   typedef DenseMap<SmallVector<const SCEV *, 4>, size_t, UniquifierDenseMapInfo>
03927     BestFormulaeTy;
03928   BestFormulaeTy BestFormulae;
03929 
03930   for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
03931     LSRUse &LU = Uses[LUIdx];
03932     DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs()); dbgs() << '\n');
03933 
03934     bool Any = false;
03935     for (size_t FIdx = 0, NumForms = LU.Formulae.size();
03936          FIdx != NumForms; ++FIdx) {
03937       Formula &F = LU.Formulae[FIdx];
03938 
03939       // Some formulas are instant losers. For example, they may depend on
03940       // nonexistent AddRecs from other loops. These need to be filtered
03941       // immediately, otherwise heuristics could choose them over others leading
03942       // to an unsatisfactory solution. Passing LoserRegs into RateFormula here
03943       // avoids the need to recompute this information across formulae using the
03944       // same bad AddRec. Passing LoserRegs is also essential unless we remove
03945       // the corresponding bad register from the Regs set.
03946       Cost CostF;
03947       Regs.clear();
03948       CostF.RateFormula(TTI, F, Regs, VisitedRegs, L, LU.Offsets, SE, DT, LU,
03949                         &LoserRegs);
03950       if (CostF.isLoser()) {
03951         // During initial formula generation, undesirable formulae are generated
03952         // by uses within other loops that have some non-trivial address mode or
03953         // use the postinc form of the IV. LSR needs to provide these formulae
03954         // as the basis of rediscovering the desired formula that uses an AddRec
03955         // corresponding to the existing phi. Once all formulae have been
03956         // generated, these initial losers may be pruned.
03957         DEBUG(dbgs() << "  Filtering loser "; F.print(dbgs());
03958               dbgs() << "\n");
03959       }
03960       else {
03961         SmallVector<const SCEV *, 4> Key;
03962         for (SmallVectorImpl<const SCEV *>::const_iterator J = F.BaseRegs.begin(),
03963                JE = F.BaseRegs.end(); J != JE; ++J) {
03964           const SCEV *Reg = *J;
03965           if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx))
03966             Key.push_back(Reg);
03967         }
03968         if (F.ScaledReg &&
03969             RegUses.isRegUsedByUsesOtherThan(F.ScaledReg, LUIdx))
03970           Key.push_back(F.ScaledReg);
03971         // Unstable sort by host order ok, because this is only used for
03972         // uniquifying.
03973         std::sort(Key.begin(), Key.end());
03974 
03975         std::pair<BestFormulaeTy::const_iterator, bool> P =
03976           BestFormulae.insert(std::make_pair(Key, FIdx));
03977         if (P.second)
03978           continue;
03979 
03980         Formula &Best = LU.Formulae[P.first->second];
03981 
03982         Cost CostBest;
03983         Regs.clear();
03984         CostBest.RateFormula(TTI, Best, Regs, VisitedRegs, L, LU.Offsets, SE,
03985                              DT, LU);
03986         if (CostF < CostBest)
03987           std::swap(F, Best);
03988         DEBUG(dbgs() << "  Filtering out formula "; F.print(dbgs());
03989               dbgs() << "\n"
03990                         "    in favor of formula "; Best.print(dbgs());
03991               dbgs() << '\n');
03992       }
03993 #ifndef NDEBUG
03994       ChangedFormulae = true;
03995 #endif
03996       LU.DeleteFormula(F);
03997       --FIdx;
03998       --NumForms;
03999       Any = true;
04000     }
04001 
04002     // Now that we've filtered out some formulae, recompute the Regs set.
04003     if (Any)
04004       LU.RecomputeRegs(LUIdx, RegUses);
04005 
04006     // Reset this to prepare for the next use.
04007     BestFormulae.clear();
04008   }
04009 
04010   DEBUG(if (ChangedFormulae) {
04011           dbgs() << "\n"
04012                     "After filtering out undesirable candidates:\n";
04013           print_uses(dbgs());
04014         });
04015 }
04016 
04017 // This is a rough guess that seems to work fairly well.
04018 static const size_t ComplexityLimit = UINT16_MAX;
04019 
04020 /// EstimateSearchSpaceComplexity - Estimate the worst-case number of
04021 /// solutions the solver might have to consider. It almost never considers
04022 /// this many solutions because it prune the search space, but the pruning
04023 /// isn't always sufficient.
04024 size_t LSRInstance::EstimateSearchSpaceComplexity() const {
04025   size_t Power = 1;
04026   for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(),
04027        E = Uses.end(); I != E; ++I) {
04028     size_t FSize = I->Formulae.size();
04029     if (FSize >= ComplexityLimit) {
04030       Power = ComplexityLimit;
04031       break;
04032     }
04033     Power *= FSize;
04034     if (Power >= ComplexityLimit)
04035       break;
04036   }
04037   return Power;
04038 }
04039 
04040 /// NarrowSearchSpaceByDetectingSupersets - When one formula uses a superset
04041 /// of the registers of another formula, it won't help reduce register
04042 /// pressure (though it may not necessarily hurt register pressure); remove
04043 /// it to simplify the system.
04044 void LSRInstance::NarrowSearchSpaceByDetectingSupersets() {
04045   if (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
04046     DEBUG(dbgs() << "The search space is too complex.\n");
04047 
04048     DEBUG(dbgs() << "Narrowing the search space by eliminating formulae "
04049                     "which use a superset of registers used by other "
04050                     "formulae.\n");
04051 
04052     for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
04053       LSRUse &LU = Uses[LUIdx];
04054       bool Any = false;
04055       for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
04056         Formula &F = LU.Formulae[i];
04057         // Look for a formula with a constant or GV in a register. If the use
04058         // also has a formula with that same value in an immediate field,
04059         // delete the one that uses a register.
04060         for (SmallVectorImpl<const SCEV *>::const_iterator
04061              I = F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) {
04062           if (const SCEVConstant *C = dyn_cast<SCEVConstant>(*I)) {
04063             Formula NewF = F;
04064             NewF.BaseOffset += C->getValue()->getSExtValue();
04065             NewF.BaseRegs.erase(NewF.BaseRegs.begin() +
04066                                 (I - F.BaseRegs.begin()));
04067             if (LU.HasFormulaWithSameRegs(NewF)) {
04068               DEBUG(dbgs() << "  Deleting "; F.print(dbgs()); dbgs() << '\n');
04069               LU.DeleteFormula(F);
04070               --i;
04071               --e;
04072               Any = true;
04073               break;
04074             }
04075           } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(*I)) {
04076             if (GlobalValue *GV = dyn_cast<GlobalValue>(U->getValue()))
04077               if (!F.BaseGV) {
04078                 Formula NewF = F;
04079                 NewF.BaseGV = GV;
04080                 NewF.BaseRegs.erase(NewF.BaseRegs.begin() +
04081                                     (I - F.BaseRegs.begin()));
04082                 if (LU.HasFormulaWithSameRegs(NewF)) {
04083                   DEBUG(dbgs() << "  Deleting "; F.print(dbgs());
04084                         dbgs() << '\n');
04085                   LU.DeleteFormula(F);
04086                   --i;
04087                   --e;
04088                   Any = true;
04089                   break;
04090                 }
04091               }
04092           }
04093         }
04094       }
04095       if (Any)
04096         LU.RecomputeRegs(LUIdx, RegUses);
04097     }
04098 
04099     DEBUG(dbgs() << "After pre-selection:\n";
04100           print_uses(dbgs()));
04101   }
04102 }
04103 
04104 /// NarrowSearchSpaceByCollapsingUnrolledCode - When there are many registers
04105 /// for expressions like A, A+1, A+2, etc., allocate a single register for
04106 /// them.
04107 void LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() {
04108   if (EstimateSearchSpaceComplexity() < ComplexityLimit)
04109     return;
04110 
04111   DEBUG(dbgs() << "The search space is too complex.\n"
04112                   "Narrowing the search space by assuming that uses separated "
04113                   "by a constant offset will use the same registers.\n");
04114 
04115   // This is especially useful for unrolled loops.
04116 
04117   for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
04118     LSRUse &LU = Uses[LUIdx];
04119     for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(),
04120          E = LU.Formulae.end(); I != E; ++I) {
04121       const Formula &F = *I;
04122       if (F.BaseOffset == 0 || (F.Scale != 0 && F.Scale != 1))
04123         continue;
04124 
04125       LSRUse *LUThatHas = FindUseWithSimilarFormula(F, LU);
04126       if (!LUThatHas)
04127         continue;
04128 
04129       if (!reconcileNewOffset(*LUThatHas, F.BaseOffset, /*HasBaseReg=*/ false,
04130                               LU.Kind, LU.AccessTy))
04131         continue;
04132 
04133       DEBUG(dbgs() << "  Deleting use "; LU.print(dbgs()); dbgs() << '\n');
04134 
04135       LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
04136 
04137       // Update the relocs to reference the new use.
04138       for (SmallVectorImpl<LSRFixup>::iterator I = Fixups.begin(),
04139            E = Fixups.end(); I != E; ++I) {
04140         LSRFixup &Fixup = *I;
04141         if (Fixup.LUIdx == LUIdx) {
04142           Fixup.LUIdx = LUThatHas - &Uses.front();
04143           Fixup.Offset += F.BaseOffset;
04144           // Add the new offset to LUThatHas' offset list.
04145           if (LUThatHas->Offsets.back() != Fixup.Offset) {
04146             LUThatHas->Offsets.push_back(Fixup.Offset);
04147             if (Fixup.Offset > LUThatHas->MaxOffset)
04148               LUThatHas->MaxOffset = Fixup.Offset;
04149             if (Fixup.Offset < LUThatHas->MinOffset)
04150               LUThatHas->MinOffset = Fixup.Offset;
04151           }
04152           DEBUG(dbgs() << "New fixup has offset " << Fixup.Offset << '\n');
04153         }
04154         if (Fixup.LUIdx == NumUses-1)
04155           Fixup.LUIdx = LUIdx;
04156       }
04157 
04158       // Delete formulae from the new use which are no longer legal.
04159       bool Any = false;
04160       for (size_t i = 0, e = LUThatHas->Formulae.size(); i != e; ++i) {
04161         Formula &F = LUThatHas->Formulae[i];
04162         if (!isLegalUse(TTI, LUThatHas->MinOffset, LUThatHas->MaxOffset,
04163                         LUThatHas->Kind, LUThatHas->AccessTy, F)) {
04164           DEBUG(dbgs() << "  Deleting "; F.print(dbgs());
04165                 dbgs() << '\n');
04166           LUThatHas->DeleteFormula(F);
04167           --i;
04168           --e;
04169           Any = true;
04170         }
04171       }
04172 
04173       if (Any)
04174         LUThatHas->RecomputeRegs(LUThatHas - &Uses.front(), RegUses);
04175 
04176       // Delete the old use.
04177       DeleteUse(LU, LUIdx);
04178       --LUIdx;
04179       --NumUses;
04180       break;
04181     }
04182   }
04183 
04184   DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
04185 }
04186 
04187 /// NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters - Call
04188 /// FilterOutUndesirableDedicatedRegisters again, if necessary, now that
04189 /// we've done more filtering, as it may be able to find more formulae to
04190 /// eliminate.
04191 void LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){
04192   if (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
04193     DEBUG(dbgs() << "The search space is too complex.\n");
04194 
04195     DEBUG(dbgs() << "Narrowing the search space by re-filtering out "
04196                     "undesirable dedicated registers.\n");
04197 
04198     FilterOutUndesirableDedicatedRegisters();
04199 
04200     DEBUG(dbgs() << "After pre-selection:\n";
04201           print_uses(dbgs()));
04202   }
04203 }
04204 
04205 /// NarrowSearchSpaceByPickingWinnerRegs - Pick a register which seems likely
04206 /// to be profitable, and then in any use which has any reference to that
04207 /// register, delete all formulae which do not reference that register.
04208 void LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() {
04209   // With all other options exhausted, loop until the system is simple
04210   // enough to handle.
04211   SmallPtrSet<const SCEV *, 4> Taken;
04212   while (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
04213     // Ok, we have too many of formulae on our hands to conveniently handle.
04214     // Use a rough heuristic to thin out the list.
04215     DEBUG(dbgs() << "The search space is too complex.\n");
04216 
04217     // Pick the register which is used by the most LSRUses, which is likely
04218     // to be a good reuse register candidate.
04219     const SCEV *Best = nullptr;
04220     unsigned BestNum = 0;
04221     for (RegUseTracker::const_iterator I = RegUses.begin(), E = RegUses.end();
04222          I != E; ++I) {
04223       const SCEV *Reg = *I;
04224       if (Taken.count(Reg))
04225         continue;
04226       if (!Best)
04227         Best = Reg;
04228       else {
04229         unsigned Count = RegUses.getUsedByIndices(Reg).count();
04230         if (Count > BestNum) {
04231           Best = Reg;
04232           BestNum = Count;
04233         }
04234       }
04235     }
04236 
04237     DEBUG(dbgs() << "Narrowing the search space by assuming " << *Best
04238                  << " will yield profitable reuse.\n");
04239     Taken.insert(Best);
04240 
04241     // In any use with formulae which references this register, delete formulae
04242     // which don't reference it.
04243     for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
04244       LSRUse &LU = Uses[LUIdx];
04245       if (!LU.Regs.count(Best)) continue;
04246 
04247       bool Any = false;
04248       for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
04249         Formula &F = LU.Formulae[i];
04250         if (!F.referencesReg(Best)) {
04251           DEBUG(dbgs() << "  Deleting "; F.print(dbgs()); dbgs() << '\n');
04252           LU.DeleteFormula(F);
04253           --e;
04254           --i;
04255           Any = true;
04256           assert(e != 0 && "Use has no formulae left! Is Regs inconsistent?");
04257           continue;
04258         }
04259       }
04260 
04261       if (Any)
04262         LU.RecomputeRegs(LUIdx, RegUses);
04263     }
04264 
04265     DEBUG(dbgs() << "After pre-selection:\n";
04266           print_uses(dbgs()));
04267   }
04268 }
04269 
04270 /// NarrowSearchSpaceUsingHeuristics - If there are an extraordinary number of
04271 /// formulae to choose from, use some rough heuristics to prune down the number
04272 /// of formulae. This keeps the main solver from taking an extraordinary amount
04273 /// of time in some worst-case scenarios.
04274 void LSRInstance::NarrowSearchSpaceUsingHeuristics() {
04275   NarrowSearchSpaceByDetectingSupersets();
04276   NarrowSearchSpaceByCollapsingUnrolledCode();
04277   NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
04278   NarrowSearchSpaceByPickingWinnerRegs();
04279 }
04280 
04281 /// SolveRecurse - This is the recursive solver.
04282 void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
04283                                Cost &SolutionCost,
04284                                SmallVectorImpl<const Formula *> &Workspace,
04285                                const Cost &CurCost,
04286                                const SmallPtrSet<const SCEV *, 16> &CurRegs,
04287                                DenseSet<const SCEV *> &VisitedRegs) const {
04288   // Some ideas:
04289   //  - prune more:
04290   //    - use more aggressive filtering
04291   //    - sort the formula so that the most profitable solutions are found first
04292   //    - sort the uses too
04293   //  - search faster:
04294   //    - don't compute a cost, and then compare. compare while computing a cost
04295   //      and bail early.
04296   //    - track register sets with SmallBitVector
04297 
04298   const LSRUse &LU = Uses[Workspace.size()];
04299 
04300   // If this use references any register that's already a part of the
04301   // in-progress solution, consider it a requirement that a formula must
04302   // reference that register in order to be considered. This prunes out
04303   // unprofitable searching.
04304   SmallSetVector<const SCEV *, 4> ReqRegs;
04305   for (SmallPtrSet<const SCEV *, 16>::const_iterator I = CurRegs.begin(),
04306        E = CurRegs.end(); I != E; ++I)
04307     if (LU.Regs.count(*I))
04308       ReqRegs.insert(*I);
04309 
04310   SmallPtrSet<const SCEV *, 16> NewRegs;
04311   Cost NewCost;
04312   for (SmallVectorImpl<Formula>::const_iterator I = LU.Formulae.begin(),
04313        E = LU.Formulae.end(); I != E; ++I) {
04314     const Formula &F = *I;
04315 
04316     // Ignore formulae which may not be ideal in terms of register reuse of
04317     // ReqRegs.  The formula should use all required registers before
04318     // introducing new ones.
04319     int NumReqRegsToFind = std::min(F.getNumRegs(), ReqRegs.size());
04320     for (SmallSetVector<const SCEV *, 4>::const_iterator J = ReqRegs.begin(),
04321          JE = ReqRegs.end(); J != JE; ++J) {
04322       const SCEV *Reg = *J;
04323       if ((F.ScaledReg && F.ScaledReg == Reg) ||
04324           std::find(F.BaseRegs.begin(), F.BaseRegs.end(), Reg) !=
04325           F.BaseRegs.end()) {
04326         --NumReqRegsToFind;
04327         if (NumReqRegsToFind == 0)
04328           break;
04329       }
04330     }
04331     if (NumReqRegsToFind != 0) {
04332       // If none of the formulae satisfied the required registers, then we could
04333       // clear ReqRegs and try again. Currently, we simply give up in this case.
04334       continue;
04335     }
04336 
04337     // Evaluate the cost of the current formula. If it's already worse than
04338     // the current best, prune the search at that point.
04339     NewCost = CurCost;
04340     NewRegs = CurRegs;
04341     NewCost.RateFormula(TTI, F, NewRegs, VisitedRegs, L, LU.Offsets, SE, DT,
04342                         LU);
04343     if (NewCost < SolutionCost) {
04344       Workspace.push_back(&F);
04345       if (Workspace.size() != Uses.size()) {
04346         SolveRecurse(Solution, SolutionCost, Workspace, NewCost,
04347                      NewRegs, VisitedRegs);
04348         if (F.getNumRegs() == 1 && Workspace.size() == 1)
04349           VisitedRegs.insert(F.ScaledReg ? F.ScaledReg : F.BaseRegs[0]);
04350       } else {
04351         DEBUG(dbgs() << "New best at "; NewCost.print(dbgs());
04352               dbgs() << ".\n Regs:";
04353               for (SmallPtrSet<const SCEV *, 16>::const_iterator
04354                    I = NewRegs.begin(), E = NewRegs.end(); I != E; ++I)
04355                 dbgs() << ' ' << **I;
04356               dbgs() << '\n');
04357 
04358         SolutionCost = NewCost;
04359         Solution = Workspace;
04360       }
04361       Workspace.pop_back();
04362     }
04363   }
04364 }
04365 
04366 /// Solve - Choose one formula from each use. Return the results in the given
04367 /// Solution vector.
04368 void LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const {
04369   SmallVector<const Formula *, 8> Workspace;
04370   Cost SolutionCost;
04371   SolutionCost.Lose();
04372   Cost CurCost;
04373   SmallPtrSet<const SCEV *, 16> CurRegs;
04374   DenseSet<const SCEV *> VisitedRegs;
04375   Workspace.reserve(Uses.size());
04376 
04377   // SolveRecurse does all the work.
04378   SolveRecurse(Solution, SolutionCost, Workspace, CurCost,
04379                CurRegs, VisitedRegs);
04380   if (Solution.empty()) {
04381     DEBUG(dbgs() << "\nNo Satisfactory Solution\n");
04382     return;
04383   }
04384 
04385   // Ok, we've now made all our decisions.
04386   DEBUG(dbgs() << "\n"
04387                   "The chosen solution requires "; SolutionCost.print(dbgs());
04388         dbgs() << ":\n";
04389         for (size_t i = 0, e = Uses.size(); i != e; ++i) {
04390           dbgs() << "  ";
04391           Uses[i].print(dbgs());
04392           dbgs() << "\n"
04393                     "    ";
04394           Solution[i]->print(dbgs());
04395           dbgs() << '\n';
04396         });
04397 
04398   assert(Solution.size() == Uses.size() && "Malformed solution!");
04399 }
04400 
04401 /// HoistInsertPosition - Helper for AdjustInsertPositionForExpand. Climb up
04402 /// the dominator tree far as we can go while still being dominated by the
04403 /// input positions. This helps canonicalize the insert position, which
04404 /// encourages sharing.
04405 BasicBlock::iterator
04406 LSRInstance::HoistInsertPosition(BasicBlock::iterator IP,
04407                                  const SmallVectorImpl<Instruction *> &Inputs)
04408                                                                          const {
04409   for (;;) {
04410     const Loop *IPLoop = LI.getLoopFor(IP->getParent());
04411     unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : 0;
04412 
04413     BasicBlock *IDom;
04414     for (DomTreeNode *Rung = DT.getNode(IP->getParent()); ; ) {
04415       if (!Rung) return IP;
04416       Rung = Rung->getIDom();
04417       if (!Rung) return IP;
04418       IDom = Rung->getBlock();
04419 
04420       // Don't climb into a loop though.
04421       const Loop *IDomLoop = LI.getLoopFor(IDom);
04422       unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : 0;
04423       if (IDomDepth <= IPLoopDepth &&
04424           (IDomDepth != IPLoopDepth || IDomLoop == IPLoop))
04425         break;
04426     }
04427 
04428     bool AllDominate = true;
04429     Instruction *BetterPos = nullptr;
04430     Instruction *Tentative = IDom->getTerminator();
04431     for (SmallVectorImpl<Instruction *>::const_iterator I = Inputs.begin(),
04432          E = Inputs.end(); I != E; ++I) {
04433       Instruction *Inst = *I;
04434       if (Inst == Tentative || !DT.dominates(Inst, Tentative)) {
04435         AllDominate = false;
04436         break;
04437       }
04438       // Attempt to find an insert position in the middle of the block,
04439       // instead of at the end, so that it can be used for other expansions.
04440       if (IDom == Inst->getParent() &&
04441           (!BetterPos || !DT.dominates(Inst, BetterPos)))
04442         BetterPos = std::next(BasicBlock::iterator(Inst));
04443     }
04444     if (!AllDominate)
04445       break;
04446     if (BetterPos)
04447       IP = BetterPos;
04448     else
04449       IP = Tentative;
04450   }
04451 
04452   return IP;
04453 }
04454 
04455 /// AdjustInsertPositionForExpand - Determine an input position which will be
04456 /// dominated by the operands and which will dominate the result.
04457 BasicBlock::iterator
04458 LSRInstance::AdjustInsertPositionForExpand(BasicBlock::iterator LowestIP,
04459                                            const LSRFixup &LF,
04460                                            const LSRUse &LU,
04461                                            SCEVExpander &Rewriter) const {
04462   // Collect some instructions which must be dominated by the
04463   // expanding replacement. These must be dominated by any operands that
04464   // will be required in the expansion.
04465   SmallVector<Instruction *, 4> Inputs;
04466   if (Instruction *I = dyn_cast<Instruction>(LF.OperandValToReplace))
04467     Inputs.push_back(I);
04468   if (LU.Kind == LSRUse::ICmpZero)
04469     if (Instruction *I =
04470           dyn_cast<Instruction>(cast<ICmpInst>(LF.UserInst)->getOperand(1)))
04471       Inputs.push_back(I);
04472   if (LF.PostIncLoops.count(L)) {
04473     if (LF.isUseFullyOutsideLoop(L))
04474       Inputs.push_back(L->getLoopLatch()->getTerminator());
04475     else
04476       Inputs.push_back(IVIncInsertPos);
04477   }
04478   // The expansion must also be dominated by the increment positions of any
04479   // loops it for which it is using post-inc mode.
04480   for (PostIncLoopSet::const_iterator I = LF.PostIncLoops.begin(),
04481        E = LF.PostIncLoops.end(); I != E; ++I) {
04482     const Loop *PIL = *I;
04483     if (PIL == L) continue;
04484 
04485     // Be dominated by the loop exit.
04486     SmallVector<BasicBlock *, 4> ExitingBlocks;
04487     PIL->getExitingBlocks(ExitingBlocks);
04488     if (!ExitingBlocks.empty()) {
04489       BasicBlock *BB = ExitingBlocks[0];
04490       for (unsigned i = 1, e = ExitingBlocks.size(); i != e; ++i)
04491         BB = DT.findNearestCommonDominator(BB, ExitingBlocks[i]);
04492       Inputs.push_back(BB->getTerminator());
04493     }
04494   }
04495 
04496   assert(!isa<PHINode>(LowestIP) && !isa<LandingPadInst>(LowestIP)
04497          && !isa<DbgInfoIntrinsic>(LowestIP) &&
04498          "Insertion point must be a normal instruction");
04499 
04500   // Then, climb up the immediate dominator tree as far as we can go while
04501   // still being dominated by the input positions.
04502   BasicBlock::iterator IP = HoistInsertPosition(LowestIP, Inputs);
04503 
04504   // Don't insert instructions before PHI nodes.
04505   while (isa<PHINode>(IP)) ++IP;
04506 
04507   // Ignore landingpad instructions.
04508   while (isa<LandingPadInst>(IP)) ++IP;
04509 
04510   // Ignore debug intrinsics.
04511   while (isa<DbgInfoIntrinsic>(IP)) ++IP;
04512 
04513   // Set IP below instructions recently inserted by SCEVExpander. This keeps the
04514   // IP consistent across expansions and allows the previously inserted
04515   // instructions to be reused by subsequent expansion.
04516   while (Rewriter.isInsertedInstruction(IP) && IP != LowestIP) ++IP;
04517 
04518   return IP;
04519 }
04520 
04521 /// Expand - Emit instructions for the leading candidate expression for this
04522 /// LSRUse (this is called "expanding").
04523 Value *LSRInstance::Expand(const LSRFixup &LF,
04524                            const Formula &F,
04525                            BasicBlock::iterator IP,
04526                            SCEVExpander &Rewriter,
04527                            SmallVectorImpl<WeakVH> &DeadInsts) const {
04528   const LSRUse &LU = Uses[LF.LUIdx];
04529   if (LU.RigidFormula)
04530     return LF.OperandValToReplace;
04531 
04532   // Determine an input position which will be dominated by the operands and
04533   // which will dominate the result.
04534   IP = AdjustInsertPositionForExpand(IP, LF, LU, Rewriter);
04535 
04536   // Inform the Rewriter if we have a post-increment use, so that it can
04537   // perform an advantageous expansion.
04538   Rewriter.setPostInc(LF.PostIncLoops);
04539 
04540   // This is the type that the user actually needs.
04541   Type *OpTy = LF.OperandValToReplace->getType();
04542   // This will be the type that we'll initially expand to.
04543   Type *Ty = F.getType();
04544   if (!Ty)
04545     // No type known; just expand directly to the ultimate type.
04546     Ty = OpTy;
04547   else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(OpTy))
04548     // Expand directly to the ultimate type if it's the right size.
04549     Ty = OpTy;
04550   // This is the type to do integer arithmetic in.
04551   Type *IntTy = SE.getEffectiveSCEVType(Ty);
04552 
04553   // Build up a list of operands to add together to form the full base.
04554   SmallVector<const SCEV *, 8> Ops;
04555 
04556   // Expand the BaseRegs portion.
04557   for (SmallVectorImpl<const SCEV *>::const_iterator I = F.BaseRegs.begin(),
04558        E = F.BaseRegs.end(); I != E; ++I) {
04559     const SCEV *Reg = *I;
04560     assert(!Reg->isZero() && "Zero allocated in a base register!");
04561 
04562     // If we're expanding for a post-inc user, make the post-inc adjustment.
04563     PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops);
04564     Reg = TransformForPostIncUse(Denormalize, Reg,
04565                                  LF.UserInst, LF.OperandValToReplace,
04566                                  Loops, SE, DT);
04567 
04568     Ops.push_back(SE.getUnknown(Rewriter.expandCodeFor(Reg, nullptr, IP)));
04569   }
04570 
04571   // Expand the ScaledReg portion.
04572   Value *ICmpScaledV = nullptr;
04573   if (F.Scale != 0) {
04574     const SCEV *ScaledS = F.ScaledReg;
04575 
04576     // If we're expanding for a post-inc user, make the post-inc adjustment.
04577     PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops);
04578     ScaledS = TransformForPostIncUse(Denormalize, ScaledS,
04579                                      LF.UserInst, LF.OperandValToReplace,
04580                                      Loops, SE, DT);
04581 
04582     if (LU.Kind == LSRUse::ICmpZero) {
04583       // Expand ScaleReg as if it was part of the base regs.
04584       if (F.Scale == 1)
04585         Ops.push_back(
04586             SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr, IP)));
04587       else {
04588         // An interesting way of "folding" with an icmp is to use a negated
04589         // scale, which we'll implement by inserting it into the other operand
04590         // of the icmp.
04591         assert(F.Scale == -1 &&
04592                "The only scale supported by ICmpZero uses is -1!");
04593         ICmpScaledV = Rewriter.expandCodeFor(ScaledS, nullptr, IP);
04594       }
04595     } else {
04596       // Otherwise just expand the scaled register and an explicit scale,
04597       // which is expected to be matched as part of the address.
04598 
04599       // Flush the operand list to suppress SCEVExpander hoisting address modes.
04600       // Unless the addressing mode will not be folded.
04601       if (!Ops.empty() && LU.Kind == LSRUse::Address &&
04602           isAMCompletelyFolded(TTI, LU, F)) {
04603         Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP);
04604         Ops.clear();
04605         Ops.push_back(SE.getUnknown(FullV));
04606       }
04607       ScaledS = SE.getUnknown(Rewriter.expandCodeFor(ScaledS, nullptr, IP));
04608       if (F.Scale != 1)
04609         ScaledS =
04610             SE.getMulExpr(ScaledS, SE.getConstant(ScaledS->getType(), F.Scale));
04611       Ops.push_back(ScaledS);
04612     }
04613   }
04614 
04615   // Expand the GV portion.
04616   if (F.BaseGV) {
04617     // Flush the operand list to suppress SCEVExpander hoisting.
04618     if (!Ops.empty()) {
04619       Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP);
04620       Ops.clear();
04621       Ops.push_back(SE.getUnknown(FullV));
04622     }
04623     Ops.push_back(SE.getUnknown(F.BaseGV));
04624   }
04625 
04626   // Flush the operand list to suppress SCEVExpander hoisting of both folded and
04627   // unfolded offsets. LSR assumes they both live next to their uses.
04628   if (!Ops.empty()) {
04629     Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty, IP);
04630     Ops.clear();
04631     Ops.push_back(SE.getUnknown(FullV));
04632   }
04633 
04634   // Expand the immediate portion.
04635   int64_t Offset = (uint64_t)F.BaseOffset + LF.Offset;
04636   if (Offset != 0) {
04637     if (LU.Kind == LSRUse::ICmpZero) {
04638       // The other interesting way of "folding" with an ICmpZero is to use a
04639       // negated immediate.
04640       if (!ICmpScaledV)
04641         ICmpScaledV = ConstantInt::get(IntTy, -(uint64_t)Offset);
04642       else {
04643         Ops.push_back(SE.getUnknown(ICmpScaledV));
04644         ICmpScaledV = ConstantInt::get(IntTy, Offset);
04645       }
04646     } else {
04647       // Just add the immediate values. These again are expected to be matched
04648       // as part of the address.
04649       Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy, Offset)));
04650     }
04651   }
04652 
04653   // Expand the unfolded offset portion.
04654   int64_t UnfoldedOffset = F.UnfoldedOffset;
04655   if (UnfoldedOffset != 0) {
04656     // Just add the immediate values.
04657     Ops.push_back(SE.getUnknown(ConstantInt::getSigned(IntTy,
04658                                                        UnfoldedOffset)));
04659   }
04660 
04661   // Emit instructions summing all the operands.
04662   const SCEV *FullS = Ops.empty() ?
04663                       SE.getConstant(IntTy, 0) :
04664                       SE.getAddExpr(Ops);
04665   Value *FullV = Rewriter.expandCodeFor(FullS, Ty, IP);
04666 
04667   // We're done expanding now, so reset the rewriter.
04668   Rewriter.clearPostInc();
04669 
04670   // An ICmpZero Formula represents an ICmp which we're handling as a
04671   // comparison against zero. Now that we've expanded an expression for that
04672   // form, update the ICmp's other operand.
04673   if (LU.Kind == LSRUse::ICmpZero) {
04674     ICmpInst *CI = cast<ICmpInst>(LF.UserInst);
04675     DeadInsts.push_back(CI->getOperand(1));
04676     assert(!F.BaseGV && "ICmp does not support folding a global value and "
04677                            "a scale at the same time!");
04678     if (F.Scale == -1) {
04679       if (ICmpScaledV->getType() != OpTy) {
04680         Instruction *Cast =
04681           CastInst::Create(CastInst::getCastOpcode(ICmpScaledV, false,
04682                                                    OpTy, false),
04683                            ICmpScaledV, OpTy, "tmp", CI);
04684         ICmpScaledV = Cast;
04685       }
04686       CI->setOperand(1, ICmpScaledV);
04687     } else {
04688       // A scale of 1 means that the scale has been expanded as part of the
04689       // base regs.
04690       assert((F.Scale == 0 || F.Scale == 1) &&
04691              "ICmp does not support folding a global value and "
04692              "a scale at the same time!");
04693       Constant *C = ConstantInt::getSigned(SE.getEffectiveSCEVType(OpTy),
04694                                            -(uint64_t)Offset);
04695       if (C->getType() != OpTy)
04696         C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false,
04697                                                           OpTy, false),
04698                                   C, OpTy);
04699 
04700       CI->setOperand(1, C);
04701     }
04702   }
04703 
04704   return FullV;
04705 }
04706 
04707 /// RewriteForPHI - Helper for Rewrite. PHI nodes are special because the use
04708 /// of their operands effectively happens in their predecessor blocks, so the
04709 /// expression may need to be expanded in multiple places.
04710 void LSRInstance::RewriteForPHI(PHINode *PN,
04711                                 const LSRFixup &LF,
04712                                 const Formula &F,
04713                                 SCEVExpander &Rewriter,
04714                                 SmallVectorImpl<WeakVH> &DeadInsts,
04715                                 Pass *P) const {
04716   DenseMap<BasicBlock *, Value *> Inserted;
04717   for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
04718     if (PN->getIncomingValue(i) == LF.OperandValToReplace) {
04719       BasicBlock *BB = PN->getIncomingBlock(i);
04720 
04721       // If this is a critical edge, split the edge so that we do not insert
04722       // the code on all predecessor/successor paths.  We do this unless this
04723       // is the canonical backedge for this loop, which complicates post-inc
04724       // users.
04725       if (e != 1 && BB->getTerminator()->getNumSuccessors() > 1 &&
04726           !isa<IndirectBrInst>(BB->getTerminator())) {
04727         BasicBlock *Parent = PN->getParent();
04728         Loop *PNLoop = LI.getLoopFor(Parent);
04729         if (!PNLoop || Parent != PNLoop->getHeader()) {
04730           // Split the critical edge.
04731           BasicBlock *NewBB = nullptr;
04732           if (!Parent->isLandingPad()) {
04733             NewBB = SplitCriticalEdge(BB, Parent, P,
04734                                       /*MergeIdenticalEdges=*/true,
04735                                       /*DontDeleteUselessPhis=*/true);
04736           } else {
04737             SmallVector<BasicBlock*, 2> NewBBs;
04738             SplitLandingPadPredecessors(Parent, BB, "", "", P, NewBBs);
04739             NewBB = NewBBs[0];
04740           }
04741           // If NewBB==NULL, then SplitCriticalEdge refused to split because all
04742           // phi predecessors are identical. The simple thing to do is skip
04743           // splitting in this case rather than complicate the API.
04744           if (NewBB) {
04745             // If PN is outside of the loop and BB is in the loop, we want to
04746             // move the block to be immediately before the PHI block, not
04747             // immediately after BB.
04748             if (L->contains(BB) && !L->contains(PN))
04749               NewBB->moveBefore(PN->getParent());
04750 
04751             // Splitting the edge can reduce the number of PHI entries we have.
04752             e = PN->getNumIncomingValues();
04753             BB = NewBB;
04754             i = PN->getBasicBlockIndex(BB);
04755           }
04756         }
04757       }
04758 
04759       std::pair<DenseMap<BasicBlock *, Value *>::iterator, bool> Pair =
04760         Inserted.insert(std::make_pair(BB, static_cast<Value *>(nullptr)));
04761       if (!Pair.second)
04762         PN->setIncomingValue(i, Pair.first->second);
04763       else {
04764         Value *FullV = Expand(LF, F, BB->getTerminator(), Rewriter, DeadInsts);
04765 
04766         // If this is reuse-by-noop-cast, insert the noop cast.
04767         Type *OpTy = LF.OperandValToReplace->getType();
04768         if (FullV->getType() != OpTy)
04769           FullV =
04770             CastInst::Create(CastInst::getCastOpcode(FullV, false,
04771                                                      OpTy, false),
04772                              FullV, LF.OperandValToReplace->getType(),
04773                              "tmp", BB->getTerminator());
04774 
04775         PN->setIncomingValue(i, FullV);
04776         Pair.first->second = FullV;
04777       }
04778     }
04779 }
04780 
04781 /// Rewrite - Emit instructions for the leading candidate expression for this
04782 /// LSRUse (this is called "expanding"), and update the UserInst to reference
04783 /// the newly expanded value.
04784 void LSRInstance::Rewrite(const LSRFixup &LF,
04785                           const Formula &F,
04786                           SCEVExpander &Rewriter,
04787                           SmallVectorImpl<WeakVH> &DeadInsts,
04788                           Pass *P) const {
04789   // First, find an insertion point that dominates UserInst. For PHI nodes,
04790   // find the nearest block which dominates all the relevant uses.
04791   if (PHINode *PN = dyn_cast<PHINode>(LF.UserInst)) {
04792     RewriteForPHI(PN, LF, F, Rewriter, DeadInsts, P);
04793   } else {
04794     Value *FullV = Expand(LF, F, LF.UserInst, Rewriter, DeadInsts);
04795 
04796     // If this is reuse-by-noop-cast, insert the noop cast.
04797     Type *OpTy = LF.OperandValToReplace->getType();
04798     if (FullV->getType() != OpTy) {
04799       Instruction *Cast =
04800         CastInst::Create(CastInst::getCastOpcode(FullV, false, OpTy, false),
04801                          FullV, OpTy, "tmp", LF.UserInst);
04802       FullV = Cast;
04803     }
04804 
04805     // Update the user. ICmpZero is handled specially here (for now) because
04806     // Expand may have updated one of the operands of the icmp already, and
04807     // its new value may happen to be equal to LF.OperandValToReplace, in
04808     // which case doing replaceUsesOfWith leads to replacing both operands
04809     // with the same value. TODO: Reorganize this.
04810     if (Uses[LF.LUIdx].Kind == LSRUse::ICmpZero)
04811       LF.UserInst->setOperand(0, FullV);
04812     else
04813       LF.UserInst->replaceUsesOfWith(LF.OperandValToReplace, FullV);
04814   }
04815 
04816   DeadInsts.push_back(LF.OperandValToReplace);
04817 }
04818 
04819 /// ImplementSolution - Rewrite all the fixup locations with new values,
04820 /// following the chosen solution.
04821 void
04822 LSRInstance::ImplementSolution(const SmallVectorImpl<const Formula *> &Solution,
04823                                Pass *P) {
04824   // Keep track of instructions we may have made dead, so that
04825   // we can remove them after we are done working.
04826   SmallVector<WeakVH, 16> DeadInsts;
04827 
04828   SCEVExpander Rewriter(SE, "lsr");
04829 #ifndef NDEBUG
04830   Rewriter.setDebugType(DEBUG_TYPE);
04831 #endif
04832   Rewriter.disableCanonicalMode();
04833   Rewriter.enableLSRMode();
04834   Rewriter.setIVIncInsertPos(L, IVIncInsertPos);
04835 
04836   // Mark phi nodes that terminate chains so the expander tries to reuse them.
04837   for (SmallVectorImpl<IVChain>::const_iterator ChainI = IVChainVec.begin(),
04838          ChainE = IVChainVec.end(); ChainI != ChainE; ++ChainI) {
04839     if (PHINode *PN = dyn_cast<PHINode>(ChainI->tailUserInst()))
04840       Rewriter.setChainedPhi(PN);
04841   }
04842 
04843   // Expand the new value definitions and update the users.
04844   for (SmallVectorImpl<LSRFixup>::const_iterator I = Fixups.begin(),
04845        E = Fixups.end(); I != E; ++I) {
04846     const LSRFixup &Fixup = *I;
04847 
04848     Rewrite(Fixup, *Solution[Fixup.LUIdx], Rewriter, DeadInsts, P);
04849 
04850     Changed = true;
04851   }
04852 
04853   for (SmallVectorImpl<IVChain>::const_iterator ChainI = IVChainVec.begin(),
04854          ChainE = IVChainVec.end(); ChainI != ChainE; ++ChainI) {
04855     GenerateIVChain(*ChainI, Rewriter, DeadInsts);
04856     Changed = true;
04857   }
04858   // Clean up after ourselves. This must be done before deleting any
04859   // instructions.
04860   Rewriter.clear();
04861 
04862   Changed |= DeleteTriviallyDeadInstructions(DeadInsts);
04863 }
04864 
04865 LSRInstance::LSRInstance(Loop *L, Pass *P)
04866     : IU(P->getAnalysis<IVUsers>()), SE(P->getAnalysis<ScalarEvolution>()),
04867       DT(P->getAnalysis<DominatorTreeWrapperPass>().getDomTree()),
04868       LI(P->getAnalysis<LoopInfo>()),
04869       TTI(P->getAnalysis<TargetTransformInfo>()), L(L), Changed(false),
04870       IVIncInsertPos(nullptr) {
04871   // If LoopSimplify form is not available, stay out of trouble.
04872   if (!L->isLoopSimplifyForm())
04873     return;
04874 
04875   // If there's no interesting work to be done, bail early.
04876   if (IU.empty()) return;
04877 
04878   // If there's too much analysis to be done, bail early. We won't be able to
04879   // model the problem anyway.
04880   unsigned NumUsers = 0;
04881   for (IVUsers::const_iterator UI = IU.begin(), E = IU.end(); UI != E; ++UI) {
04882     if (++NumUsers > MaxIVUsers) {
04883       DEBUG(dbgs() << "LSR skipping loop, too many IV Users in " << *L
04884             << "\n");
04885       return;
04886     }
04887   }
04888 
04889 #ifndef NDEBUG
04890   // All dominating loops must have preheaders, or SCEVExpander may not be able
04891   // to materialize an AddRecExpr whose Start is an outer AddRecExpr.
04892   //
04893   // IVUsers analysis should only create users that are dominated by simple loop
04894   // headers. Since this loop should dominate all of its users, its user list
04895   // should be empty if this loop itself is not within a simple loop nest.
04896   for (DomTreeNode *Rung = DT.getNode(L->getLoopPreheader());
04897        Rung; Rung = Rung->getIDom()) {
04898     BasicBlock *BB = Rung->getBlock();
04899     const Loop *DomLoop = LI.getLoopFor(BB);
04900     if (DomLoop && DomLoop->getHeader() == BB) {
04901       assert(DomLoop->getLoopPreheader() && "LSR needs a simplified loop nest");
04902     }
04903   }
04904 #endif // DEBUG
04905 
04906   DEBUG(dbgs() << "\nLSR on loop ";
04907         L->getHeader()->printAsOperand(dbgs(), /*PrintType=*/false);
04908         dbgs() << ":\n");
04909 
04910   // First, perform some low-level loop optimizations.
04911   OptimizeShadowIV();
04912   OptimizeLoopTermCond();
04913 
04914   // If loop preparation eliminates all interesting IV users, bail.
04915   if (IU.empty()) return;
04916 
04917   // Skip nested loops until we can model them better with formulae.
04918   if (!L->empty()) {
04919     DEBUG(dbgs() << "LSR skipping outer loop " << *L << "\n");
04920     return;
04921   }
04922 
04923   // Start collecting data and preparing for the solver.
04924   CollectChains();
04925   CollectInterestingTypesAndFactors();
04926   CollectFixupsAndInitialFormulae();
04927   CollectLoopInvariantFixupsAndFormulae();
04928 
04929   assert(!Uses.empty() && "IVUsers reported at least one use");
04930   DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n";
04931         print_uses(dbgs()));
04932 
04933   // Now use the reuse data to generate a bunch of interesting ways
04934   // to formulate the values needed for the uses.
04935   GenerateAllReuseFormulae();
04936 
04937   FilterOutUndesirableDedicatedRegisters();
04938   NarrowSearchSpaceUsingHeuristics();
04939 
04940   SmallVector<const Formula *, 8> Solution;
04941   Solve(Solution);
04942 
04943   // Release memory that is no longer needed.
04944   Factors.clear();
04945   Types.clear();
04946   RegUses.clear();
04947 
04948   if (Solution.empty())
04949     return;
04950 
04951 #ifndef NDEBUG
04952   // Formulae should be legal.
04953   for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(), E = Uses.end();
04954        I != E; ++I) {
04955     const LSRUse &LU = *I;
04956     for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(),
04957                                                   JE = LU.Formulae.end();
04958          J != JE; ++J)
04959       assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
04960                         *J) && "Illegal formula generated!");
04961   };
04962 #endif
04963 
04964   // Now that we've decided what we want, make it so.
04965   ImplementSolution(Solution, P);
04966 }
04967 
04968 void LSRInstance::print_factors_and_types(raw_ostream &OS) const {
04969   if (Factors.empty() && Types.empty()) return;
04970 
04971   OS << "LSR has identified the following interesting factors and types: ";
04972   bool First = true;
04973 
04974   for (SmallSetVector<int64_t, 8>::const_iterator
04975        I = Factors.begin(), E = Factors.end(); I != E; ++I) {
04976     if (!First) OS << ", ";
04977     First = false;
04978     OS << '*' << *I;
04979   }
04980 
04981   for (SmallSetVector<Type *, 4>::const_iterator
04982        I = Types.begin(), E = Types.end(); I != E; ++I) {
04983     if (!First) OS << ", ";
04984     First = false;
04985     OS << '(' << **I << ')';
04986   }
04987   OS << '\n';
04988 }
04989 
04990 void LSRInstance::print_fixups(raw_ostream &OS) const {
04991   OS << "LSR is examining the following fixup sites:\n";
04992   for (SmallVectorImpl<LSRFixup>::const_iterator I = Fixups.begin(),
04993        E = Fixups.end(); I != E; ++I) {
04994     dbgs() << "  ";
04995     I->print(OS);
04996     OS << '\n';
04997   }
04998 }
04999 
05000 void LSRInstance::print_uses(raw_ostream &OS) const {
05001   OS << "LSR is examining the following uses:\n";
05002   for (SmallVectorImpl<LSRUse>::const_iterator I = Uses.begin(),
05003        E = Uses.end(); I != E; ++I) {
05004     const LSRUse &LU = *I;
05005     dbgs() << "  ";
05006     LU.print(OS);
05007     OS << '\n';
05008     for (SmallVectorImpl<Formula>::const_iterator J = LU.Formulae.begin(),
05009          JE = LU.Formulae.end(); J != JE; ++J) {
05010       OS << "    ";
05011       J->print(OS);
05012       OS << '\n';
05013     }
05014   }
05015 }
05016 
05017 void LSRInstance::print(raw_ostream &OS) const {
05018   print_factors_and_types(OS);
05019   print_fixups(OS);
05020   print_uses(OS);
05021 }
05022 
05023 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
05024 void LSRInstance::dump() const {
05025   print(errs()); errs() << '\n';
05026 }
05027 #endif
05028 
05029 namespace {
05030 
05031 class LoopStrengthReduce : public LoopPass {
05032 public:
05033   static char ID; // Pass ID, replacement for typeid
05034   LoopStrengthReduce();
05035 
05036 private:
05037   bool runOnLoop(Loop *L, LPPassManager &LPM) override;
05038   void getAnalysisUsage(AnalysisUsage &AU) const override;
05039 };
05040 
05041 }
05042 
05043 char LoopStrengthReduce::ID = 0;
05044 INITIALIZE_PASS_BEGIN(LoopStrengthReduce, "loop-reduce",
05045                 "Loop Strength Reduction", false, false)
05046 INITIALIZE_AG_DEPENDENCY(TargetTransformInfo)
05047 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
05048 INITIALIZE_PASS_DEPENDENCY(ScalarEvolution)
05049 INITIALIZE_PASS_DEPENDENCY(IVUsers)
05050 INITIALIZE_PASS_DEPENDENCY(LoopInfo)
05051 INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
05052 INITIALIZE_PASS_END(LoopStrengthReduce, "loop-reduce",
05053                 "Loop Strength Reduction", false, false)
05054 
05055 
05056 Pass *llvm::createLoopStrengthReducePass() {
05057   return new LoopStrengthReduce();
05058 }
05059 
05060 LoopStrengthReduce::LoopStrengthReduce() : LoopPass(ID) {
05061   initializeLoopStrengthReducePass(*PassRegistry::getPassRegistry());
05062 }
05063 
05064 void LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const {
05065   // We split critical edges, so we change the CFG.  However, we do update
05066   // many analyses if they are around.
05067   AU.addPreservedID(LoopSimplifyID);
05068 
05069   AU.addRequired<LoopInfo>();
05070   AU.addPreserved<LoopInfo>();
05071   AU.addRequiredID(LoopSimplifyID);
05072   AU.addRequired<DominatorTreeWrapperPass>();
05073   AU.addPreserved<DominatorTreeWrapperPass>();
05074   AU.addRequired<ScalarEvolution>();
05075   AU.addPreserved<ScalarEvolution>();
05076   // Requiring LoopSimplify a second time here prevents IVUsers from running
05077   // twice, since LoopSimplify was invalidated by running ScalarEvolution.
05078   AU.addRequiredID(LoopSimplifyID);
05079   AU.addRequired<IVUsers>();
05080   AU.addPreserved<IVUsers>();
05081   AU.addRequired<TargetTransformInfo>();
05082 }
05083 
05084 bool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager & /*LPM*/) {
05085   if (skipOptnoneFunction(L))
05086     return false;
05087 
05088   bool Changed = false;
05089 
05090   // Run the main LSR transformation.
05091   Changed |= LSRInstance(L, this).getChanged();
05092 
05093   // Remove any extra phis created by processing inner loops.
05094   Changed |= DeleteDeadPHIs(L->getHeader());
05095   if (EnablePhiElim && L->isLoopSimplifyForm()) {
05096     SmallVector<WeakVH, 16> DeadInsts;
05097     SCEVExpander Rewriter(getAnalysis<ScalarEvolution>(), "lsr");
05098 #ifndef NDEBUG
05099     Rewriter.setDebugType(DEBUG_TYPE);
05100 #endif
05101     unsigned numFolded = Rewriter.replaceCongruentIVs(
05102         L, &getAnalysis<DominatorTreeWrapperPass>().getDomTree(), DeadInsts,
05103         &getAnalysis<TargetTransformInfo>());
05104     if (numFolded) {
05105       Changed = true;
05106       DeleteTriviallyDeadInstructions(DeadInsts);
05107       DeleteDeadPHIs(L->getHeader());
05108     }
05109   }
05110   return Changed;
05111 }