LLVM API Documentation

ScalarEvolution.cpp
Go to the documentation of this file.
00001 //===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file contains the implementation of the scalar evolution analysis
00011 // engine, which is used primarily to analyze expressions involving induction
00012 // variables in loops.
00013 //
00014 // There are several aspects to this library.  First is the representation of
00015 // scalar expressions, which are represented as subclasses of the SCEV class.
00016 // These classes are used to represent certain types of subexpressions that we
00017 // can handle. We only create one SCEV of a particular shape, so
00018 // pointer-comparisons for equality are legal.
00019 //
00020 // One important aspect of the SCEV objects is that they are never cyclic, even
00021 // if there is a cycle in the dataflow for an expression (ie, a PHI node).  If
00022 // the PHI node is one of the idioms that we can represent (e.g., a polynomial
00023 // recurrence) then we represent it directly as a recurrence node, otherwise we
00024 // represent it as a SCEVUnknown node.
00025 //
00026 // In addition to being able to represent expressions of various types, we also
00027 // have folders that are used to build the *canonical* representation for a
00028 // particular expression.  These folders are capable of using a variety of
00029 // rewrite rules to simplify the expressions.
00030 //
00031 // Once the folders are defined, we can implement the more interesting
00032 // higher-level code, such as the code that recognizes PHI nodes of various
00033 // types, computes the execution count of a loop, etc.
00034 //
00035 // TODO: We should use these routines and value representations to implement
00036 // dependence analysis!
00037 //
00038 //===----------------------------------------------------------------------===//
00039 //
00040 // There are several good references for the techniques used in this analysis.
00041 //
00042 //  Chains of recurrences -- a method to expedite the evaluation
00043 //  of closed-form functions
00044 //  Olaf Bachmann, Paul S. Wang, Eugene V. Zima
00045 //
00046 //  On computational properties of chains of recurrences
00047 //  Eugene V. Zima
00048 //
00049 //  Symbolic Evaluation of Chains of Recurrences for Loop Optimization
00050 //  Robert A. van Engelen
00051 //
00052 //  Efficient Symbolic Analysis for Optimizing Compilers
00053 //  Robert A. van Engelen
00054 //
00055 //  Using the chains of recurrences algebra for data dependence testing and
00056 //  induction variable substitution
00057 //  MS Thesis, Johnie Birch
00058 //
00059 //===----------------------------------------------------------------------===//
00060 
00061 #include "llvm/Analysis/ScalarEvolution.h"
00062 #include "llvm/ADT/Optional.h"
00063 #include "llvm/ADT/STLExtras.h"
00064 #include "llvm/ADT/SmallPtrSet.h"
00065 #include "llvm/ADT/Statistic.h"
00066 #include "llvm/Analysis/AssumptionTracker.h"
00067 #include "llvm/Analysis/ConstantFolding.h"
00068 #include "llvm/Analysis/InstructionSimplify.h"
00069 #include "llvm/Analysis/LoopInfo.h"
00070 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
00071 #include "llvm/Analysis/ValueTracking.h"
00072 #include "llvm/IR/ConstantRange.h"
00073 #include "llvm/IR/Constants.h"
00074 #include "llvm/IR/DataLayout.h"
00075 #include "llvm/IR/DerivedTypes.h"
00076 #include "llvm/IR/Dominators.h"
00077 #include "llvm/IR/GetElementPtrTypeIterator.h"
00078 #include "llvm/IR/GlobalAlias.h"
00079 #include "llvm/IR/GlobalVariable.h"
00080 #include "llvm/IR/InstIterator.h"
00081 #include "llvm/IR/Instructions.h"
00082 #include "llvm/IR/LLVMContext.h"
00083 #include "llvm/IR/Metadata.h"
00084 #include "llvm/IR/Operator.h"
00085 #include "llvm/Support/CommandLine.h"
00086 #include "llvm/Support/Debug.h"
00087 #include "llvm/Support/ErrorHandling.h"
00088 #include "llvm/Support/MathExtras.h"
00089 #include "llvm/Support/raw_ostream.h"
00090 #include "llvm/Target/TargetLibraryInfo.h"
00091 #include <algorithm>
00092 using namespace llvm;
00093 
00094 #define DEBUG_TYPE "scalar-evolution"
00095 
00096 STATISTIC(NumArrayLenItCounts,
00097           "Number of trip counts computed with array length");
00098 STATISTIC(NumTripCountsComputed,
00099           "Number of loops with predictable loop counts");
00100 STATISTIC(NumTripCountsNotComputed,
00101           "Number of loops without predictable loop counts");
00102 STATISTIC(NumBruteForceTripCountsComputed,
00103           "Number of loops with trip counts computed by force");
00104 
00105 static cl::opt<unsigned>
00106 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
00107                         cl::desc("Maximum number of iterations SCEV will "
00108                                  "symbolically execute a constant "
00109                                  "derived loop"),
00110                         cl::init(100));
00111 
00112 // FIXME: Enable this with XDEBUG when the test suite is clean.
00113 static cl::opt<bool>
00114 VerifySCEV("verify-scev",
00115            cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
00116 
00117 INITIALIZE_PASS_BEGIN(ScalarEvolution, "scalar-evolution",
00118                 "Scalar Evolution Analysis", false, true)
00119 INITIALIZE_PASS_DEPENDENCY(AssumptionTracker)
00120 INITIALIZE_PASS_DEPENDENCY(LoopInfo)
00121 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
00122 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfo)
00123 INITIALIZE_PASS_END(ScalarEvolution, "scalar-evolution",
00124                 "Scalar Evolution Analysis", false, true)
00125 char ScalarEvolution::ID = 0;
00126 
00127 //===----------------------------------------------------------------------===//
00128 //                           SCEV class definitions
00129 //===----------------------------------------------------------------------===//
00130 
00131 //===----------------------------------------------------------------------===//
00132 // Implementation of the SCEV class.
00133 //
00134 
00135 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
00136 void SCEV::dump() const {
00137   print(dbgs());
00138   dbgs() << '\n';
00139 }
00140 #endif
00141 
00142 void SCEV::print(raw_ostream &OS) const {
00143   switch (static_cast<SCEVTypes>(getSCEVType())) {
00144   case scConstant:
00145     cast<SCEVConstant>(this)->getValue()->printAsOperand(OS, false);
00146     return;
00147   case scTruncate: {
00148     const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this);
00149     const SCEV *Op = Trunc->getOperand();
00150     OS << "(trunc " << *Op->getType() << " " << *Op << " to "
00151        << *Trunc->getType() << ")";
00152     return;
00153   }
00154   case scZeroExtend: {
00155     const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(this);
00156     const SCEV *Op = ZExt->getOperand();
00157     OS << "(zext " << *Op->getType() << " " << *Op << " to "
00158        << *ZExt->getType() << ")";
00159     return;
00160   }
00161   case scSignExtend: {
00162     const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(this);
00163     const SCEV *Op = SExt->getOperand();
00164     OS << "(sext " << *Op->getType() << " " << *Op << " to "
00165        << *SExt->getType() << ")";
00166     return;
00167   }
00168   case scAddRecExpr: {
00169     const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(this);
00170     OS << "{" << *AR->getOperand(0);
00171     for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
00172       OS << ",+," << *AR->getOperand(i);
00173     OS << "}<";
00174     if (AR->getNoWrapFlags(FlagNUW))
00175       OS << "nuw><";
00176     if (AR->getNoWrapFlags(FlagNSW))
00177       OS << "nsw><";
00178     if (AR->getNoWrapFlags(FlagNW) &&
00179         !AR->getNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW)))
00180       OS << "nw><";
00181     AR->getLoop()->getHeader()->printAsOperand(OS, /*PrintType=*/false);
00182     OS << ">";
00183     return;
00184   }
00185   case scAddExpr:
00186   case scMulExpr:
00187   case scUMaxExpr:
00188   case scSMaxExpr: {
00189     const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(this);
00190     const char *OpStr = nullptr;
00191     switch (NAry->getSCEVType()) {
00192     case scAddExpr: OpStr = " + "; break;
00193     case scMulExpr: OpStr = " * "; break;
00194     case scUMaxExpr: OpStr = " umax "; break;
00195     case scSMaxExpr: OpStr = " smax "; break;
00196     }
00197     OS << "(";
00198     for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end();
00199          I != E; ++I) {
00200       OS << **I;
00201       if (std::next(I) != E)
00202         OS << OpStr;
00203     }
00204     OS << ")";
00205     switch (NAry->getSCEVType()) {
00206     case scAddExpr:
00207     case scMulExpr:
00208       if (NAry->getNoWrapFlags(FlagNUW))
00209         OS << "<nuw>";
00210       if (NAry->getNoWrapFlags(FlagNSW))
00211         OS << "<nsw>";
00212     }
00213     return;
00214   }
00215   case scUDivExpr: {
00216     const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this);
00217     OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")";
00218     return;
00219   }
00220   case scUnknown: {
00221     const SCEVUnknown *U = cast<SCEVUnknown>(this);
00222     Type *AllocTy;
00223     if (U->isSizeOf(AllocTy)) {
00224       OS << "sizeof(" << *AllocTy << ")";
00225       return;
00226     }
00227     if (U->isAlignOf(AllocTy)) {
00228       OS << "alignof(" << *AllocTy << ")";
00229       return;
00230     }
00231 
00232     Type *CTy;
00233     Constant *FieldNo;
00234     if (U->isOffsetOf(CTy, FieldNo)) {
00235       OS << "offsetof(" << *CTy << ", ";
00236       FieldNo->printAsOperand(OS, false);
00237       OS << ")";
00238       return;
00239     }
00240 
00241     // Otherwise just print it normally.
00242     U->getValue()->printAsOperand(OS, false);
00243     return;
00244   }
00245   case scCouldNotCompute:
00246     OS << "***COULDNOTCOMPUTE***";
00247     return;
00248   }
00249   llvm_unreachable("Unknown SCEV kind!");
00250 }
00251 
00252 Type *SCEV::getType() const {
00253   switch (static_cast<SCEVTypes>(getSCEVType())) {
00254   case scConstant:
00255     return cast<SCEVConstant>(this)->getType();
00256   case scTruncate:
00257   case scZeroExtend:
00258   case scSignExtend:
00259     return cast<SCEVCastExpr>(this)->getType();
00260   case scAddRecExpr:
00261   case scMulExpr:
00262   case scUMaxExpr:
00263   case scSMaxExpr:
00264     return cast<SCEVNAryExpr>(this)->getType();
00265   case scAddExpr:
00266     return cast<SCEVAddExpr>(this)->getType();
00267   case scUDivExpr:
00268     return cast<SCEVUDivExpr>(this)->getType();
00269   case scUnknown:
00270     return cast<SCEVUnknown>(this)->getType();
00271   case scCouldNotCompute:
00272     llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
00273   }
00274   llvm_unreachable("Unknown SCEV kind!");
00275 }
00276 
00277 bool SCEV::isZero() const {
00278   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
00279     return SC->getValue()->isZero();
00280   return false;
00281 }
00282 
00283 bool SCEV::isOne() const {
00284   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
00285     return SC->getValue()->isOne();
00286   return false;
00287 }
00288 
00289 bool SCEV::isAllOnesValue() const {
00290   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
00291     return SC->getValue()->isAllOnesValue();
00292   return false;
00293 }
00294 
00295 /// isNonConstantNegative - Return true if the specified scev is negated, but
00296 /// not a constant.
00297 bool SCEV::isNonConstantNegative() const {
00298   const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(this);
00299   if (!Mul) return false;
00300 
00301   // If there is a constant factor, it will be first.
00302   const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0));
00303   if (!SC) return false;
00304 
00305   // Return true if the value is negative, this matches things like (-42 * V).
00306   return SC->getValue()->getValue().isNegative();
00307 }
00308 
00309 SCEVCouldNotCompute::SCEVCouldNotCompute() :
00310   SCEV(FoldingSetNodeIDRef(), scCouldNotCompute) {}
00311 
00312 bool SCEVCouldNotCompute::classof(const SCEV *S) {
00313   return S->getSCEVType() == scCouldNotCompute;
00314 }
00315 
00316 const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
00317   FoldingSetNodeID ID;
00318   ID.AddInteger(scConstant);
00319   ID.AddPointer(V);
00320   void *IP = nullptr;
00321   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
00322   SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V);
00323   UniqueSCEVs.InsertNode(S, IP);
00324   return S;
00325 }
00326 
00327 const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
00328   return getConstant(ConstantInt::get(getContext(), Val));
00329 }
00330 
00331 const SCEV *
00332 ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
00333   IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty));
00334   return getConstant(ConstantInt::get(ITy, V, isSigned));
00335 }
00336 
00337 SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID,
00338                            unsigned SCEVTy, const SCEV *op, Type *ty)
00339   : SCEV(ID, SCEVTy), Op(op), Ty(ty) {}
00340 
00341 SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID,
00342                                    const SCEV *op, Type *ty)
00343   : SCEVCastExpr(ID, scTruncate, op, ty) {
00344   assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
00345          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
00346          "Cannot truncate non-integer value!");
00347 }
00348 
00349 SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID,
00350                                        const SCEV *op, Type *ty)
00351   : SCEVCastExpr(ID, scZeroExtend, op, ty) {
00352   assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
00353          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
00354          "Cannot zero extend non-integer value!");
00355 }
00356 
00357 SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID,
00358                                        const SCEV *op, Type *ty)
00359   : SCEVCastExpr(ID, scSignExtend, op, ty) {
00360   assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
00361          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
00362          "Cannot sign extend non-integer value!");
00363 }
00364 
00365 void SCEVUnknown::deleted() {
00366   // Clear this SCEVUnknown from various maps.
00367   SE->forgetMemoizedResults(this);
00368 
00369   // Remove this SCEVUnknown from the uniquing map.
00370   SE->UniqueSCEVs.RemoveNode(this);
00371 
00372   // Release the value.
00373   setValPtr(nullptr);
00374 }
00375 
00376 void SCEVUnknown::allUsesReplacedWith(Value *New) {
00377   // Clear this SCEVUnknown from various maps.
00378   SE->forgetMemoizedResults(this);
00379 
00380   // Remove this SCEVUnknown from the uniquing map.
00381   SE->UniqueSCEVs.RemoveNode(this);
00382 
00383   // Update this SCEVUnknown to point to the new value. This is needed
00384   // because there may still be outstanding SCEVs which still point to
00385   // this SCEVUnknown.
00386   setValPtr(New);
00387 }
00388 
00389 bool SCEVUnknown::isSizeOf(Type *&AllocTy) const {
00390   if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
00391     if (VCE->getOpcode() == Instruction::PtrToInt)
00392       if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
00393         if (CE->getOpcode() == Instruction::GetElementPtr &&
00394             CE->getOperand(0)->isNullValue() &&
00395             CE->getNumOperands() == 2)
00396           if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(1)))
00397             if (CI->isOne()) {
00398               AllocTy = cast<PointerType>(CE->getOperand(0)->getType())
00399                                  ->getElementType();
00400               return true;
00401             }
00402 
00403   return false;
00404 }
00405 
00406 bool SCEVUnknown::isAlignOf(Type *&AllocTy) const {
00407   if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
00408     if (VCE->getOpcode() == Instruction::PtrToInt)
00409       if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
00410         if (CE->getOpcode() == Instruction::GetElementPtr &&
00411             CE->getOperand(0)->isNullValue()) {
00412           Type *Ty =
00413             cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
00414           if (StructType *STy = dyn_cast<StructType>(Ty))
00415             if (!STy->isPacked() &&
00416                 CE->getNumOperands() == 3 &&
00417                 CE->getOperand(1)->isNullValue()) {
00418               if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(2)))
00419                 if (CI->isOne() &&
00420                     STy->getNumElements() == 2 &&
00421                     STy->getElementType(0)->isIntegerTy(1)) {
00422                   AllocTy = STy->getElementType(1);
00423                   return true;
00424                 }
00425             }
00426         }
00427 
00428   return false;
00429 }
00430 
00431 bool SCEVUnknown::isOffsetOf(Type *&CTy, Constant *&FieldNo) const {
00432   if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
00433     if (VCE->getOpcode() == Instruction::PtrToInt)
00434       if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
00435         if (CE->getOpcode() == Instruction::GetElementPtr &&
00436             CE->getNumOperands() == 3 &&
00437             CE->getOperand(0)->isNullValue() &&
00438             CE->getOperand(1)->isNullValue()) {
00439           Type *Ty =
00440             cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
00441           // Ignore vector types here so that ScalarEvolutionExpander doesn't
00442           // emit getelementptrs that index into vectors.
00443           if (Ty->isStructTy() || Ty->isArrayTy()) {
00444             CTy = Ty;
00445             FieldNo = CE->getOperand(2);
00446             return true;
00447           }
00448         }
00449 
00450   return false;
00451 }
00452 
00453 //===----------------------------------------------------------------------===//
00454 //                               SCEV Utilities
00455 //===----------------------------------------------------------------------===//
00456 
00457 namespace {
00458   /// SCEVComplexityCompare - Return true if the complexity of the LHS is less
00459   /// than the complexity of the RHS.  This comparator is used to canonicalize
00460   /// expressions.
00461   class SCEVComplexityCompare {
00462     const LoopInfo *const LI;
00463   public:
00464     explicit SCEVComplexityCompare(const LoopInfo *li) : LI(li) {}
00465 
00466     // Return true or false if LHS is less than, or at least RHS, respectively.
00467     bool operator()(const SCEV *LHS, const SCEV *RHS) const {
00468       return compare(LHS, RHS) < 0;
00469     }
00470 
00471     // Return negative, zero, or positive, if LHS is less than, equal to, or
00472     // greater than RHS, respectively. A three-way result allows recursive
00473     // comparisons to be more efficient.
00474     int compare(const SCEV *LHS, const SCEV *RHS) const {
00475       // Fast-path: SCEVs are uniqued so we can do a quick equality check.
00476       if (LHS == RHS)
00477         return 0;
00478 
00479       // Primarily, sort the SCEVs by their getSCEVType().
00480       unsigned LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
00481       if (LType != RType)
00482         return (int)LType - (int)RType;
00483 
00484       // Aside from the getSCEVType() ordering, the particular ordering
00485       // isn't very important except that it's beneficial to be consistent,
00486       // so that (a + b) and (b + a) don't end up as different expressions.
00487       switch (static_cast<SCEVTypes>(LType)) {
00488       case scUnknown: {
00489         const SCEVUnknown *LU = cast<SCEVUnknown>(LHS);
00490         const SCEVUnknown *RU = cast<SCEVUnknown>(RHS);
00491 
00492         // Sort SCEVUnknown values with some loose heuristics. TODO: This is
00493         // not as complete as it could be.
00494         const Value *LV = LU->getValue(), *RV = RU->getValue();
00495 
00496         // Order pointer values after integer values. This helps SCEVExpander
00497         // form GEPs.
00498         bool LIsPointer = LV->getType()->isPointerTy(),
00499              RIsPointer = RV->getType()->isPointerTy();
00500         if (LIsPointer != RIsPointer)
00501           return (int)LIsPointer - (int)RIsPointer;
00502 
00503         // Compare getValueID values.
00504         unsigned LID = LV->getValueID(),
00505                  RID = RV->getValueID();
00506         if (LID != RID)
00507           return (int)LID - (int)RID;
00508 
00509         // Sort arguments by their position.
00510         if (const Argument *LA = dyn_cast<Argument>(LV)) {
00511           const Argument *RA = cast<Argument>(RV);
00512           unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
00513           return (int)LArgNo - (int)RArgNo;
00514         }
00515 
00516         // For instructions, compare their loop depth, and their operand
00517         // count.  This is pretty loose.
00518         if (const Instruction *LInst = dyn_cast<Instruction>(LV)) {
00519           const Instruction *RInst = cast<Instruction>(RV);
00520 
00521           // Compare loop depths.
00522           const BasicBlock *LParent = LInst->getParent(),
00523                            *RParent = RInst->getParent();
00524           if (LParent != RParent) {
00525             unsigned LDepth = LI->getLoopDepth(LParent),
00526                      RDepth = LI->getLoopDepth(RParent);
00527             if (LDepth != RDepth)
00528               return (int)LDepth - (int)RDepth;
00529           }
00530 
00531           // Compare the number of operands.
00532           unsigned LNumOps = LInst->getNumOperands(),
00533                    RNumOps = RInst->getNumOperands();
00534           return (int)LNumOps - (int)RNumOps;
00535         }
00536 
00537         return 0;
00538       }
00539 
00540       case scConstant: {
00541         const SCEVConstant *LC = cast<SCEVConstant>(LHS);
00542         const SCEVConstant *RC = cast<SCEVConstant>(RHS);
00543 
00544         // Compare constant values.
00545         const APInt &LA = LC->getValue()->getValue();
00546         const APInt &RA = RC->getValue()->getValue();
00547         unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
00548         if (LBitWidth != RBitWidth)
00549           return (int)LBitWidth - (int)RBitWidth;
00550         return LA.ult(RA) ? -1 : 1;
00551       }
00552 
00553       case scAddRecExpr: {
00554         const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(LHS);
00555         const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(RHS);
00556 
00557         // Compare addrec loop depths.
00558         const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
00559         if (LLoop != RLoop) {
00560           unsigned LDepth = LLoop->getLoopDepth(),
00561                    RDepth = RLoop->getLoopDepth();
00562           if (LDepth != RDepth)
00563             return (int)LDepth - (int)RDepth;
00564         }
00565 
00566         // Addrec complexity grows with operand count.
00567         unsigned LNumOps = LA->getNumOperands(), RNumOps = RA->getNumOperands();
00568         if (LNumOps != RNumOps)
00569           return (int)LNumOps - (int)RNumOps;
00570 
00571         // Lexicographically compare.
00572         for (unsigned i = 0; i != LNumOps; ++i) {
00573           long X = compare(LA->getOperand(i), RA->getOperand(i));
00574           if (X != 0)
00575             return X;
00576         }
00577 
00578         return 0;
00579       }
00580 
00581       case scAddExpr:
00582       case scMulExpr:
00583       case scSMaxExpr:
00584       case scUMaxExpr: {
00585         const SCEVNAryExpr *LC = cast<SCEVNAryExpr>(LHS);
00586         const SCEVNAryExpr *RC = cast<SCEVNAryExpr>(RHS);
00587 
00588         // Lexicographically compare n-ary expressions.
00589         unsigned LNumOps = LC->getNumOperands(), RNumOps = RC->getNumOperands();
00590         if (LNumOps != RNumOps)
00591           return (int)LNumOps - (int)RNumOps;
00592 
00593         for (unsigned i = 0; i != LNumOps; ++i) {
00594           if (i >= RNumOps)
00595             return 1;
00596           long X = compare(LC->getOperand(i), RC->getOperand(i));
00597           if (X != 0)
00598             return X;
00599         }
00600         return (int)LNumOps - (int)RNumOps;
00601       }
00602 
00603       case scUDivExpr: {
00604         const SCEVUDivExpr *LC = cast<SCEVUDivExpr>(LHS);
00605         const SCEVUDivExpr *RC = cast<SCEVUDivExpr>(RHS);
00606 
00607         // Lexicographically compare udiv expressions.
00608         long X = compare(LC->getLHS(), RC->getLHS());
00609         if (X != 0)
00610           return X;
00611         return compare(LC->getRHS(), RC->getRHS());
00612       }
00613 
00614       case scTruncate:
00615       case scZeroExtend:
00616       case scSignExtend: {
00617         const SCEVCastExpr *LC = cast<SCEVCastExpr>(LHS);
00618         const SCEVCastExpr *RC = cast<SCEVCastExpr>(RHS);
00619 
00620         // Compare cast expressions by operand.
00621         return compare(LC->getOperand(), RC->getOperand());
00622       }
00623 
00624       case scCouldNotCompute:
00625         llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
00626       }
00627       llvm_unreachable("Unknown SCEV kind!");
00628     }
00629   };
00630 }
00631 
00632 /// GroupByComplexity - Given a list of SCEV objects, order them by their
00633 /// complexity, and group objects of the same complexity together by value.
00634 /// When this routine is finished, we know that any duplicates in the vector are
00635 /// consecutive and that complexity is monotonically increasing.
00636 ///
00637 /// Note that we go take special precautions to ensure that we get deterministic
00638 /// results from this routine.  In other words, we don't want the results of
00639 /// this to depend on where the addresses of various SCEV objects happened to
00640 /// land in memory.
00641 ///
00642 static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops,
00643                               LoopInfo *LI) {
00644   if (Ops.size() < 2) return;  // Noop
00645   if (Ops.size() == 2) {
00646     // This is the common case, which also happens to be trivially simple.
00647     // Special case it.
00648     const SCEV *&LHS = Ops[0], *&RHS = Ops[1];
00649     if (SCEVComplexityCompare(LI)(RHS, LHS))
00650       std::swap(LHS, RHS);
00651     return;
00652   }
00653 
00654   // Do the rough sort by complexity.
00655   std::stable_sort(Ops.begin(), Ops.end(), SCEVComplexityCompare(LI));
00656 
00657   // Now that we are sorted by complexity, group elements of the same
00658   // complexity.  Note that this is, at worst, N^2, but the vector is likely to
00659   // be extremely short in practice.  Note that we take this approach because we
00660   // do not want to depend on the addresses of the objects we are grouping.
00661   for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
00662     const SCEV *S = Ops[i];
00663     unsigned Complexity = S->getSCEVType();
00664 
00665     // If there are any objects of the same complexity and same value as this
00666     // one, group them.
00667     for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
00668       if (Ops[j] == S) { // Found a duplicate.
00669         // Move it to immediately after i'th element.
00670         std::swap(Ops[i+1], Ops[j]);
00671         ++i;   // no need to rescan it.
00672         if (i == e-2) return;  // Done!
00673       }
00674     }
00675   }
00676 }
00677 
00678 static const APInt srem(const SCEVConstant *C1, const SCEVConstant *C2) {
00679   APInt A = C1->getValue()->getValue();
00680   APInt B = C2->getValue()->getValue();
00681   uint32_t ABW = A.getBitWidth();
00682   uint32_t BBW = B.getBitWidth();
00683 
00684   if (ABW > BBW)
00685     B = B.sext(ABW);
00686   else if (ABW < BBW)
00687     A = A.sext(BBW);
00688 
00689   return APIntOps::srem(A, B);
00690 }
00691 
00692 static const APInt sdiv(const SCEVConstant *C1, const SCEVConstant *C2) {
00693   APInt A = C1->getValue()->getValue();
00694   APInt B = C2->getValue()->getValue();
00695   uint32_t ABW = A.getBitWidth();
00696   uint32_t BBW = B.getBitWidth();
00697 
00698   if (ABW > BBW)
00699     B = B.sext(ABW);
00700   else if (ABW < BBW)
00701     A = A.sext(BBW);
00702 
00703   return APIntOps::sdiv(A, B);
00704 }
00705 
00706 static const APInt urem(const SCEVConstant *C1, const SCEVConstant *C2) {
00707   APInt A = C1->getValue()->getValue();
00708   APInt B = C2->getValue()->getValue();
00709   uint32_t ABW = A.getBitWidth();
00710   uint32_t BBW = B.getBitWidth();
00711 
00712   if (ABW > BBW)
00713     B = B.zext(ABW);
00714   else if (ABW < BBW)
00715     A = A.zext(BBW);
00716 
00717   return APIntOps::urem(A, B);
00718 }
00719 
00720 static const APInt udiv(const SCEVConstant *C1, const SCEVConstant *C2) {
00721   APInt A = C1->getValue()->getValue();
00722   APInt B = C2->getValue()->getValue();
00723   uint32_t ABW = A.getBitWidth();
00724   uint32_t BBW = B.getBitWidth();
00725 
00726   if (ABW > BBW)
00727     B = B.zext(ABW);
00728   else if (ABW < BBW)
00729     A = A.zext(BBW);
00730 
00731   return APIntOps::udiv(A, B);
00732 }
00733 
00734 namespace {
00735 struct FindSCEVSize {
00736   int Size;
00737   FindSCEVSize() : Size(0) {}
00738 
00739   bool follow(const SCEV *S) {
00740     ++Size;
00741     // Keep looking at all operands of S.
00742     return true;
00743   }
00744   bool isDone() const {
00745     return false;
00746   }
00747 };
00748 }
00749 
00750 // Returns the size of the SCEV S.
00751 static inline int sizeOfSCEV(const SCEV *S) {
00752   FindSCEVSize F;
00753   SCEVTraversal<FindSCEVSize> ST(F);
00754   ST.visitAll(S);
00755   return F.Size;
00756 }
00757 
00758 namespace {
00759 
00760 template <typename Derived>
00761 struct SCEVDivision : public SCEVVisitor<Derived, void> {
00762 public:
00763   // Computes the Quotient and Remainder of the division of Numerator by
00764   // Denominator.
00765   static void divide(ScalarEvolution &SE, const SCEV *Numerator,
00766                      const SCEV *Denominator, const SCEV **Quotient,
00767                      const SCEV **Remainder) {
00768     assert(Numerator && Denominator && "Uninitialized SCEV");
00769 
00770     Derived D(SE, Numerator, Denominator);
00771 
00772     // Check for the trivial case here to avoid having to check for it in the
00773     // rest of the code.
00774     if (Numerator == Denominator) {
00775       *Quotient = D.One;
00776       *Remainder = D.Zero;
00777       return;
00778     }
00779 
00780     if (Numerator->isZero()) {
00781       *Quotient = D.Zero;
00782       *Remainder = D.Zero;
00783       return;
00784     }
00785 
00786     // Split the Denominator when it is a product.
00787     if (const SCEVMulExpr *T = dyn_cast<const SCEVMulExpr>(Denominator)) {
00788       const SCEV *Q, *R;
00789       *Quotient = Numerator;
00790       for (const SCEV *Op : T->operands()) {
00791         divide(SE, *Quotient, Op, &Q, &R);
00792         *Quotient = Q;
00793 
00794         // Bail out when the Numerator is not divisible by one of the terms of
00795         // the Denominator.
00796         if (!R->isZero()) {
00797           *Quotient = D.Zero;
00798           *Remainder = Numerator;
00799           return;
00800         }
00801       }
00802       *Remainder = D.Zero;
00803       return;
00804     }
00805 
00806     D.visit(Numerator);
00807     *Quotient = D.Quotient;
00808     *Remainder = D.Remainder;
00809   }
00810 
00811   // Except in the trivial case described above, we do not know how to divide
00812   // Expr by Denominator for the following functions with empty implementation.
00813   void visitTruncateExpr(const SCEVTruncateExpr *Numerator) {}
00814   void visitZeroExtendExpr(const SCEVZeroExtendExpr *Numerator) {}
00815   void visitSignExtendExpr(const SCEVSignExtendExpr *Numerator) {}
00816   void visitUDivExpr(const SCEVUDivExpr *Numerator) {}
00817   void visitSMaxExpr(const SCEVSMaxExpr *Numerator) {}
00818   void visitUMaxExpr(const SCEVUMaxExpr *Numerator) {}
00819   void visitUnknown(const SCEVUnknown *Numerator) {}
00820   void visitCouldNotCompute(const SCEVCouldNotCompute *Numerator) {}
00821 
00822   void visitAddRecExpr(const SCEVAddRecExpr *Numerator) {
00823     const SCEV *StartQ, *StartR, *StepQ, *StepR;
00824     assert(Numerator->isAffine() && "Numerator should be affine");
00825     divide(SE, Numerator->getStart(), Denominator, &StartQ, &StartR);
00826     divide(SE, Numerator->getStepRecurrence(SE), Denominator, &StepQ, &StepR);
00827     Quotient = SE.getAddRecExpr(StartQ, StepQ, Numerator->getLoop(),
00828                                 Numerator->getNoWrapFlags());
00829     Remainder = SE.getAddRecExpr(StartR, StepR, Numerator->getLoop(),
00830                                  Numerator->getNoWrapFlags());
00831   }
00832 
00833   void visitAddExpr(const SCEVAddExpr *Numerator) {
00834     SmallVector<const SCEV *, 2> Qs, Rs;
00835     Type *Ty = Denominator->getType();
00836 
00837     for (const SCEV *Op : Numerator->operands()) {
00838       const SCEV *Q, *R;
00839       divide(SE, Op, Denominator, &Q, &R);
00840 
00841       // Bail out if types do not match.
00842       if (Ty != Q->getType() || Ty != R->getType()) {
00843         Quotient = Zero;
00844         Remainder = Numerator;
00845         return;
00846       }
00847 
00848       Qs.push_back(Q);
00849       Rs.push_back(R);
00850     }
00851 
00852     if (Qs.size() == 1) {
00853       Quotient = Qs[0];
00854       Remainder = Rs[0];
00855       return;
00856     }
00857 
00858     Quotient = SE.getAddExpr(Qs);
00859     Remainder = SE.getAddExpr(Rs);
00860   }
00861 
00862   void visitMulExpr(const SCEVMulExpr *Numerator) {
00863     SmallVector<const SCEV *, 2> Qs;
00864     Type *Ty = Denominator->getType();
00865 
00866     bool FoundDenominatorTerm = false;
00867     for (const SCEV *Op : Numerator->operands()) {
00868       // Bail out if types do not match.
00869       if (Ty != Op->getType()) {
00870         Quotient = Zero;
00871         Remainder = Numerator;
00872         return;
00873       }
00874 
00875       if (FoundDenominatorTerm) {
00876         Qs.push_back(Op);
00877         continue;
00878       }
00879 
00880       // Check whether Denominator divides one of the product operands.
00881       const SCEV *Q, *R;
00882       divide(SE, Op, Denominator, &Q, &R);
00883       if (!R->isZero()) {
00884         Qs.push_back(Op);
00885         continue;
00886       }
00887 
00888       // Bail out if types do not match.
00889       if (Ty != Q->getType()) {
00890         Quotient = Zero;
00891         Remainder = Numerator;
00892         return;
00893       }
00894 
00895       FoundDenominatorTerm = true;
00896       Qs.push_back(Q);
00897     }
00898 
00899     if (FoundDenominatorTerm) {
00900       Remainder = Zero;
00901       if (Qs.size() == 1)
00902         Quotient = Qs[0];
00903       else
00904         Quotient = SE.getMulExpr(Qs);
00905       return;
00906     }
00907 
00908     if (!isa<SCEVUnknown>(Denominator)) {
00909       Quotient = Zero;
00910       Remainder = Numerator;
00911       return;
00912     }
00913 
00914     // The Remainder is obtained by replacing Denominator by 0 in Numerator.
00915     ValueToValueMap RewriteMap;
00916     RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] =
00917         cast<SCEVConstant>(Zero)->getValue();
00918     Remainder = SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true);
00919 
00920     if (Remainder->isZero()) {
00921       // The Quotient is obtained by replacing Denominator by 1 in Numerator.
00922       RewriteMap[cast<SCEVUnknown>(Denominator)->getValue()] =
00923           cast<SCEVConstant>(One)->getValue();
00924       Quotient =
00925           SCEVParameterRewriter::rewrite(Numerator, SE, RewriteMap, true);
00926       return;
00927     }
00928 
00929     // Quotient is (Numerator - Remainder) divided by Denominator.
00930     const SCEV *Q, *R;
00931     const SCEV *Diff = SE.getMinusSCEV(Numerator, Remainder);
00932     if (sizeOfSCEV(Diff) > sizeOfSCEV(Numerator)) {
00933       // This SCEV does not seem to simplify: fail the division here.
00934       Quotient = Zero;
00935       Remainder = Numerator;
00936       return;
00937     }
00938     divide(SE, Diff, Denominator, &Q, &R);
00939     assert(R == Zero &&
00940            "(Numerator - Remainder) should evenly divide Denominator");
00941     Quotient = Q;
00942   }
00943 
00944 private:
00945   SCEVDivision(ScalarEvolution &S, const SCEV *Numerator,
00946                const SCEV *Denominator)
00947       : SE(S), Denominator(Denominator) {
00948     Zero = SE.getConstant(Denominator->getType(), 0);
00949     One = SE.getConstant(Denominator->getType(), 1);
00950 
00951     // By default, we don't know how to divide Expr by Denominator.
00952     // Providing the default here simplifies the rest of the code.
00953     Quotient = Zero;
00954     Remainder = Numerator;
00955   }
00956 
00957   ScalarEvolution &SE;
00958   const SCEV *Denominator, *Quotient, *Remainder, *Zero, *One;
00959 
00960   friend struct SCEVSDivision;
00961   friend struct SCEVUDivision;
00962 };
00963 
00964 struct SCEVSDivision : public SCEVDivision<SCEVSDivision> {
00965   SCEVSDivision(ScalarEvolution &S, const SCEV *Numerator,
00966                 const SCEV *Denominator)
00967       : SCEVDivision(S, Numerator, Denominator) {}
00968 
00969   void visitConstant(const SCEVConstant *Numerator) {
00970     if (const SCEVConstant *D = dyn_cast<SCEVConstant>(Denominator)) {
00971       Quotient = SE.getConstant(sdiv(Numerator, D));
00972       Remainder = SE.getConstant(srem(Numerator, D));
00973       return;
00974     }
00975   }
00976 };
00977 
00978 struct SCEVUDivision : public SCEVDivision<SCEVUDivision> {
00979   SCEVUDivision(ScalarEvolution &S, const SCEV *Numerator,
00980                 const SCEV *Denominator)
00981       : SCEVDivision(S, Numerator, Denominator) {}
00982 
00983   void visitConstant(const SCEVConstant *Numerator) {
00984     if (const SCEVConstant *D = dyn_cast<SCEVConstant>(Denominator)) {
00985       Quotient = SE.getConstant(udiv(Numerator, D));
00986       Remainder = SE.getConstant(urem(Numerator, D));
00987       return;
00988     }
00989   }
00990 };
00991 
00992 }
00993 
00994 //===----------------------------------------------------------------------===//
00995 //                      Simple SCEV method implementations
00996 //===----------------------------------------------------------------------===//
00997 
00998 /// BinomialCoefficient - Compute BC(It, K).  The result has width W.
00999 /// Assume, K > 0.
01000 static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
01001                                        ScalarEvolution &SE,
01002                                        Type *ResultTy) {
01003   // Handle the simplest case efficiently.
01004   if (K == 1)
01005     return SE.getTruncateOrZeroExtend(It, ResultTy);
01006 
01007   // We are using the following formula for BC(It, K):
01008   //
01009   //   BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
01010   //
01011   // Suppose, W is the bitwidth of the return value.  We must be prepared for
01012   // overflow.  Hence, we must assure that the result of our computation is
01013   // equal to the accurate one modulo 2^W.  Unfortunately, division isn't
01014   // safe in modular arithmetic.
01015   //
01016   // However, this code doesn't use exactly that formula; the formula it uses
01017   // is something like the following, where T is the number of factors of 2 in
01018   // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
01019   // exponentiation:
01020   //
01021   //   BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
01022   //
01023   // This formula is trivially equivalent to the previous formula.  However,
01024   // this formula can be implemented much more efficiently.  The trick is that
01025   // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
01026   // arithmetic.  To do exact division in modular arithmetic, all we have
01027   // to do is multiply by the inverse.  Therefore, this step can be done at
01028   // width W.
01029   //
01030   // The next issue is how to safely do the division by 2^T.  The way this
01031   // is done is by doing the multiplication step at a width of at least W + T
01032   // bits.  This way, the bottom W+T bits of the product are accurate. Then,
01033   // when we perform the division by 2^T (which is equivalent to a right shift
01034   // by T), the bottom W bits are accurate.  Extra bits are okay; they'll get
01035   // truncated out after the division by 2^T.
01036   //
01037   // In comparison to just directly using the first formula, this technique
01038   // is much more efficient; using the first formula requires W * K bits,
01039   // but this formula less than W + K bits. Also, the first formula requires
01040   // a division step, whereas this formula only requires multiplies and shifts.
01041   //
01042   // It doesn't matter whether the subtraction step is done in the calculation
01043   // width or the input iteration count's width; if the subtraction overflows,
01044   // the result must be zero anyway.  We prefer here to do it in the width of
01045   // the induction variable because it helps a lot for certain cases; CodeGen
01046   // isn't smart enough to ignore the overflow, which leads to much less
01047   // efficient code if the width of the subtraction is wider than the native
01048   // register width.
01049   //
01050   // (It's possible to not widen at all by pulling out factors of 2 before
01051   // the multiplication; for example, K=2 can be calculated as
01052   // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
01053   // extra arithmetic, so it's not an obvious win, and it gets
01054   // much more complicated for K > 3.)
01055 
01056   // Protection from insane SCEVs; this bound is conservative,
01057   // but it probably doesn't matter.
01058   if (K > 1000)
01059     return SE.getCouldNotCompute();
01060 
01061   unsigned W = SE.getTypeSizeInBits(ResultTy);
01062 
01063   // Calculate K! / 2^T and T; we divide out the factors of two before
01064   // multiplying for calculating K! / 2^T to avoid overflow.
01065   // Other overflow doesn't matter because we only care about the bottom
01066   // W bits of the result.
01067   APInt OddFactorial(W, 1);
01068   unsigned T = 1;
01069   for (unsigned i = 3; i <= K; ++i) {
01070     APInt Mult(W, i);
01071     unsigned TwoFactors = Mult.countTrailingZeros();
01072     T += TwoFactors;
01073     Mult = Mult.lshr(TwoFactors);
01074     OddFactorial *= Mult;
01075   }
01076 
01077   // We need at least W + T bits for the multiplication step
01078   unsigned CalculationBits = W + T;
01079 
01080   // Calculate 2^T, at width T+W.
01081   APInt DivFactor = APInt::getOneBitSet(CalculationBits, T);
01082 
01083   // Calculate the multiplicative inverse of K! / 2^T;
01084   // this multiplication factor will perform the exact division by
01085   // K! / 2^T.
01086   APInt Mod = APInt::getSignedMinValue(W+1);
01087   APInt MultiplyFactor = OddFactorial.zext(W+1);
01088   MultiplyFactor = MultiplyFactor.multiplicativeInverse(Mod);
01089   MultiplyFactor = MultiplyFactor.trunc(W);
01090 
01091   // Calculate the product, at width T+W
01092   IntegerType *CalculationTy = IntegerType::get(SE.getContext(),
01093                                                       CalculationBits);
01094   const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy);
01095   for (unsigned i = 1; i != K; ++i) {
01096     const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i));
01097     Dividend = SE.getMulExpr(Dividend,
01098                              SE.getTruncateOrZeroExtend(S, CalculationTy));
01099   }
01100 
01101   // Divide by 2^T
01102   const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor));
01103 
01104   // Truncate the result, and divide by K! / 2^T.
01105 
01106   return SE.getMulExpr(SE.getConstant(MultiplyFactor),
01107                        SE.getTruncateOrZeroExtend(DivResult, ResultTy));
01108 }
01109 
01110 /// evaluateAtIteration - Return the value of this chain of recurrences at
01111 /// the specified iteration number.  We can evaluate this recurrence by
01112 /// multiplying each element in the chain by the binomial coefficient
01113 /// corresponding to it.  In other words, we can evaluate {A,+,B,+,C,+,D} as:
01114 ///
01115 ///   A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
01116 ///
01117 /// where BC(It, k) stands for binomial coefficient.
01118 ///
01119 const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
01120                                                 ScalarEvolution &SE) const {
01121   const SCEV *Result = getStart();
01122   for (unsigned i = 1, e = getNumOperands(); i != e; ++i) {
01123     // The computation is correct in the face of overflow provided that the
01124     // multiplication is performed _after_ the evaluation of the binomial
01125     // coefficient.
01126     const SCEV *Coeff = BinomialCoefficient(It, i, SE, getType());
01127     if (isa<SCEVCouldNotCompute>(Coeff))
01128       return Coeff;
01129 
01130     Result = SE.getAddExpr(Result, SE.getMulExpr(getOperand(i), Coeff));
01131   }
01132   return Result;
01133 }
01134 
01135 //===----------------------------------------------------------------------===//
01136 //                    SCEV Expression folder implementations
01137 //===----------------------------------------------------------------------===//
01138 
01139 const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op,
01140                                              Type *Ty) {
01141   assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
01142          "This is not a truncating conversion!");
01143   assert(isSCEVable(Ty) &&
01144          "This is not a conversion to a SCEVable type!");
01145   Ty = getEffectiveSCEVType(Ty);
01146 
01147   FoldingSetNodeID ID;
01148   ID.AddInteger(scTruncate);
01149   ID.AddPointer(Op);
01150   ID.AddPointer(Ty);
01151   void *IP = nullptr;
01152   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01153 
01154   // Fold if the operand is constant.
01155   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
01156     return getConstant(
01157       cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(), Ty)));
01158 
01159   // trunc(trunc(x)) --> trunc(x)
01160   if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op))
01161     return getTruncateExpr(ST->getOperand(), Ty);
01162 
01163   // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
01164   if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op))
01165     return getTruncateOrSignExtend(SS->getOperand(), Ty);
01166 
01167   // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
01168   if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
01169     return getTruncateOrZeroExtend(SZ->getOperand(), Ty);
01170 
01171   // trunc(x1+x2+...+xN) --> trunc(x1)+trunc(x2)+...+trunc(xN) if we can
01172   // eliminate all the truncates.
01173   if (const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Op)) {
01174     SmallVector<const SCEV *, 4> Operands;
01175     bool hasTrunc = false;
01176     for (unsigned i = 0, e = SA->getNumOperands(); i != e && !hasTrunc; ++i) {
01177       const SCEV *S = getTruncateExpr(SA->getOperand(i), Ty);
01178       hasTrunc = isa<SCEVTruncateExpr>(S);
01179       Operands.push_back(S);
01180     }
01181     if (!hasTrunc)
01182       return getAddExpr(Operands);
01183     UniqueSCEVs.FindNodeOrInsertPos(ID, IP);  // Mutates IP, returns NULL.
01184   }
01185 
01186   // trunc(x1*x2*...*xN) --> trunc(x1)*trunc(x2)*...*trunc(xN) if we can
01187   // eliminate all the truncates.
01188   if (const SCEVMulExpr *SM = dyn_cast<SCEVMulExpr>(Op)) {
01189     SmallVector<const SCEV *, 4> Operands;
01190     bool hasTrunc = false;
01191     for (unsigned i = 0, e = SM->getNumOperands(); i != e && !hasTrunc; ++i) {
01192       const SCEV *S = getTruncateExpr(SM->getOperand(i), Ty);
01193       hasTrunc = isa<SCEVTruncateExpr>(S);
01194       Operands.push_back(S);
01195     }
01196     if (!hasTrunc)
01197       return getMulExpr(Operands);
01198     UniqueSCEVs.FindNodeOrInsertPos(ID, IP);  // Mutates IP, returns NULL.
01199   }
01200 
01201   // If the input value is a chrec scev, truncate the chrec's operands.
01202   if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) {
01203     SmallVector<const SCEV *, 4> Operands;
01204     for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i)
01205       Operands.push_back(getTruncateExpr(AddRec->getOperand(i), Ty));
01206     return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap);
01207   }
01208 
01209   // The cast wasn't folded; create an explicit cast node. We can reuse
01210   // the existing insert position since if we get here, we won't have
01211   // made any changes which would invalidate it.
01212   SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator),
01213                                                  Op, Ty);
01214   UniqueSCEVs.InsertNode(S, IP);
01215   return S;
01216 }
01217 
01218 const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op,
01219                                                Type *Ty) {
01220   assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
01221          "This is not an extending conversion!");
01222   assert(isSCEVable(Ty) &&
01223          "This is not a conversion to a SCEVable type!");
01224   Ty = getEffectiveSCEVType(Ty);
01225 
01226   // Fold if the operand is constant.
01227   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
01228     return getConstant(
01229       cast<ConstantInt>(ConstantExpr::getZExt(SC->getValue(), Ty)));
01230 
01231   // zext(zext(x)) --> zext(x)
01232   if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
01233     return getZeroExtendExpr(SZ->getOperand(), Ty);
01234 
01235   // Before doing any expensive analysis, check to see if we've already
01236   // computed a SCEV for this Op and Ty.
01237   FoldingSetNodeID ID;
01238   ID.AddInteger(scZeroExtend);
01239   ID.AddPointer(Op);
01240   ID.AddPointer(Ty);
01241   void *IP = nullptr;
01242   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01243 
01244   // zext(trunc(x)) --> zext(x) or x or trunc(x)
01245   if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) {
01246     // It's possible the bits taken off by the truncate were all zero bits. If
01247     // so, we should be able to simplify this further.
01248     const SCEV *X = ST->getOperand();
01249     ConstantRange CR = getUnsignedRange(X);
01250     unsigned TruncBits = getTypeSizeInBits(ST->getType());
01251     unsigned NewBits = getTypeSizeInBits(Ty);
01252     if (CR.truncate(TruncBits).zeroExtend(NewBits).contains(
01253             CR.zextOrTrunc(NewBits)))
01254       return getTruncateOrZeroExtend(X, Ty);
01255   }
01256 
01257   // If the input value is a chrec scev, and we can prove that the value
01258   // did not overflow the old, smaller, value, we can zero extend all of the
01259   // operands (often constants).  This allows analysis of something like
01260   // this:  for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
01261   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op))
01262     if (AR->isAffine()) {
01263       const SCEV *Start = AR->getStart();
01264       const SCEV *Step = AR->getStepRecurrence(*this);
01265       unsigned BitWidth = getTypeSizeInBits(AR->getType());
01266       const Loop *L = AR->getLoop();
01267 
01268       // If we have special knowledge that this addrec won't overflow,
01269       // we don't need to do any further analysis.
01270       if (AR->getNoWrapFlags(SCEV::FlagNUW))
01271         return getAddRecExpr(getZeroExtendExpr(Start, Ty),
01272                              getZeroExtendExpr(Step, Ty),
01273                              L, AR->getNoWrapFlags());
01274 
01275       // Check whether the backedge-taken count is SCEVCouldNotCompute.
01276       // Note that this serves two purposes: It filters out loops that are
01277       // simply not analyzable, and it covers the case where this code is
01278       // being called from within backedge-taken count analysis, such that
01279       // attempting to ask for the backedge-taken count would likely result
01280       // in infinite recursion. In the later case, the analysis code will
01281       // cope with a conservative value, and it will take care to purge
01282       // that value once it has finished.
01283       const SCEV *MaxBECount = getMaxBackedgeTakenCount(L);
01284       if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
01285         // Manually compute the final value for AR, checking for
01286         // overflow.
01287 
01288         // Check whether the backedge-taken count can be losslessly casted to
01289         // the addrec's type. The count is always unsigned.
01290         const SCEV *CastedMaxBECount =
01291           getTruncateOrZeroExtend(MaxBECount, Start->getType());
01292         const SCEV *RecastedMaxBECount =
01293           getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType());
01294         if (MaxBECount == RecastedMaxBECount) {
01295           Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
01296           // Check whether Start+Step*MaxBECount has no unsigned overflow.
01297           const SCEV *ZMul = getMulExpr(CastedMaxBECount, Step);
01298           const SCEV *ZAdd = getZeroExtendExpr(getAddExpr(Start, ZMul), WideTy);
01299           const SCEV *WideStart = getZeroExtendExpr(Start, WideTy);
01300           const SCEV *WideMaxBECount =
01301             getZeroExtendExpr(CastedMaxBECount, WideTy);
01302           const SCEV *OperandExtendedAdd =
01303             getAddExpr(WideStart,
01304                        getMulExpr(WideMaxBECount,
01305                                   getZeroExtendExpr(Step, WideTy)));
01306           if (ZAdd == OperandExtendedAdd) {
01307             // Cache knowledge of AR NUW, which is propagated to this AddRec.
01308             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW);
01309             // Return the expression with the addrec on the outside.
01310             return getAddRecExpr(getZeroExtendExpr(Start, Ty),
01311                                  getZeroExtendExpr(Step, Ty),
01312                                  L, AR->getNoWrapFlags());
01313           }
01314           // Similar to above, only this time treat the step value as signed.
01315           // This covers loops that count down.
01316           OperandExtendedAdd =
01317             getAddExpr(WideStart,
01318                        getMulExpr(WideMaxBECount,
01319                                   getSignExtendExpr(Step, WideTy)));
01320           if (ZAdd == OperandExtendedAdd) {
01321             // Cache knowledge of AR NW, which is propagated to this AddRec.
01322             // Negative step causes unsigned wrap, but it still can't self-wrap.
01323             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
01324             // Return the expression with the addrec on the outside.
01325             return getAddRecExpr(getZeroExtendExpr(Start, Ty),
01326                                  getSignExtendExpr(Step, Ty),
01327                                  L, AR->getNoWrapFlags());
01328           }
01329         }
01330 
01331         // If the backedge is guarded by a comparison with the pre-inc value
01332         // the addrec is safe. Also, if the entry is guarded by a comparison
01333         // with the start value and the backedge is guarded by a comparison
01334         // with the post-inc value, the addrec is safe.
01335         if (isKnownPositive(Step)) {
01336           const SCEV *N = getConstant(APInt::getMinValue(BitWidth) -
01337                                       getUnsignedRange(Step).getUnsignedMax());
01338           if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, AR, N) ||
01339               (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_ULT, Start, N) &&
01340                isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT,
01341                                            AR->getPostIncExpr(*this), N))) {
01342             // Cache knowledge of AR NUW, which is propagated to this AddRec.
01343             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW);
01344             // Return the expression with the addrec on the outside.
01345             return getAddRecExpr(getZeroExtendExpr(Start, Ty),
01346                                  getZeroExtendExpr(Step, Ty),
01347                                  L, AR->getNoWrapFlags());
01348           }
01349         } else if (isKnownNegative(Step)) {
01350           const SCEV *N = getConstant(APInt::getMaxValue(BitWidth) -
01351                                       getSignedRange(Step).getSignedMin());
01352           if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, AR, N) ||
01353               (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_UGT, Start, N) &&
01354                isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT,
01355                                            AR->getPostIncExpr(*this), N))) {
01356             // Cache knowledge of AR NW, which is propagated to this AddRec.
01357             // Negative step causes unsigned wrap, but it still can't self-wrap.
01358             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
01359             // Return the expression with the addrec on the outside.
01360             return getAddRecExpr(getZeroExtendExpr(Start, Ty),
01361                                  getSignExtendExpr(Step, Ty),
01362                                  L, AR->getNoWrapFlags());
01363           }
01364         }
01365       }
01366     }
01367 
01368   // The cast wasn't folded; create an explicit cast node.
01369   // Recompute the insert position, as it may have been invalidated.
01370   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01371   SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
01372                                                    Op, Ty);
01373   UniqueSCEVs.InsertNode(S, IP);
01374   return S;
01375 }
01376 
01377 // Get the limit of a recurrence such that incrementing by Step cannot cause
01378 // signed overflow as long as the value of the recurrence within the loop does
01379 // not exceed this limit before incrementing.
01380 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
01381                                            ICmpInst::Predicate *Pred,
01382                                            ScalarEvolution *SE) {
01383   unsigned BitWidth = SE->getTypeSizeInBits(Step->getType());
01384   if (SE->isKnownPositive(Step)) {
01385     *Pred = ICmpInst::ICMP_SLT;
01386     return SE->getConstant(APInt::getSignedMinValue(BitWidth) -
01387                            SE->getSignedRange(Step).getSignedMax());
01388   }
01389   if (SE->isKnownNegative(Step)) {
01390     *Pred = ICmpInst::ICMP_SGT;
01391     return SE->getConstant(APInt::getSignedMaxValue(BitWidth) -
01392                        SE->getSignedRange(Step).getSignedMin());
01393   }
01394   return nullptr;
01395 }
01396 
01397 // The recurrence AR has been shown to have no signed wrap. Typically, if we can
01398 // prove NSW for AR, then we can just as easily prove NSW for its preincrement
01399 // or postincrement sibling. This allows normalizing a sign extended AddRec as
01400 // such: {sext(Step + Start),+,Step} => {(Step + sext(Start),+,Step} As a
01401 // result, the expression "Step + sext(PreIncAR)" is congruent with
01402 // "sext(PostIncAR)"
01403 static const SCEV *getPreStartForSignExtend(const SCEVAddRecExpr *AR,
01404                                             Type *Ty,
01405                                             ScalarEvolution *SE) {
01406   const Loop *L = AR->getLoop();
01407   const SCEV *Start = AR->getStart();
01408   const SCEV *Step = AR->getStepRecurrence(*SE);
01409 
01410   // Check for a simple looking step prior to loop entry.
01411   const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start);
01412   if (!SA)
01413     return nullptr;
01414 
01415   // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
01416   // subtraction is expensive. For this purpose, perform a quick and dirty
01417   // difference, by checking for Step in the operand list.
01418   SmallVector<const SCEV *, 4> DiffOps;
01419   for (const SCEV *Op : SA->operands())
01420     if (Op != Step)
01421       DiffOps.push_back(Op);
01422 
01423   if (DiffOps.size() == SA->getNumOperands())
01424     return nullptr;
01425 
01426   // This is a postinc AR. Check for overflow on the preinc recurrence using the
01427   // same three conditions that getSignExtendedExpr checks.
01428 
01429   // 1. NSW flags on the step increment.
01430   const SCEV *PreStart = SE->getAddExpr(DiffOps, SA->getNoWrapFlags());
01431   const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
01432     SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagAnyWrap));
01433 
01434   if (PreAR && PreAR->getNoWrapFlags(SCEV::FlagNSW))
01435     return PreStart;
01436 
01437   // 2. Direct overflow check on the step operation's expression.
01438   unsigned BitWidth = SE->getTypeSizeInBits(AR->getType());
01439   Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2);
01440   const SCEV *OperandExtendedStart =
01441     SE->getAddExpr(SE->getSignExtendExpr(PreStart, WideTy),
01442                    SE->getSignExtendExpr(Step, WideTy));
01443   if (SE->getSignExtendExpr(Start, WideTy) == OperandExtendedStart) {
01444     // Cache knowledge of PreAR NSW.
01445     if (PreAR)
01446       const_cast<SCEVAddRecExpr *>(PreAR)->setNoWrapFlags(SCEV::FlagNSW);
01447     // FIXME: this optimization needs a unit test
01448     DEBUG(dbgs() << "SCEV: untested prestart overflow check\n");
01449     return PreStart;
01450   }
01451 
01452   // 3. Loop precondition.
01453   ICmpInst::Predicate Pred;
01454   const SCEV *OverflowLimit = getOverflowLimitForStep(Step, &Pred, SE);
01455 
01456   if (OverflowLimit &&
01457       SE->isLoopEntryGuardedByCond(L, Pred, PreStart, OverflowLimit)) {
01458     return PreStart;
01459   }
01460   return nullptr;
01461 }
01462 
01463 // Get the normalized sign-extended expression for this AddRec's Start.
01464 static const SCEV *getSignExtendAddRecStart(const SCEVAddRecExpr *AR,
01465                                             Type *Ty,
01466                                             ScalarEvolution *SE) {
01467   const SCEV *PreStart = getPreStartForSignExtend(AR, Ty, SE);
01468   if (!PreStart)
01469     return SE->getSignExtendExpr(AR->getStart(), Ty);
01470 
01471   return SE->getAddExpr(SE->getSignExtendExpr(AR->getStepRecurrence(*SE), Ty),
01472                         SE->getSignExtendExpr(PreStart, Ty));
01473 }
01474 
01475 const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op,
01476                                                Type *Ty) {
01477   assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
01478          "This is not an extending conversion!");
01479   assert(isSCEVable(Ty) &&
01480          "This is not a conversion to a SCEVable type!");
01481   Ty = getEffectiveSCEVType(Ty);
01482 
01483   // Fold if the operand is constant.
01484   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
01485     return getConstant(
01486       cast<ConstantInt>(ConstantExpr::getSExt(SC->getValue(), Ty)));
01487 
01488   // sext(sext(x)) --> sext(x)
01489   if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op))
01490     return getSignExtendExpr(SS->getOperand(), Ty);
01491 
01492   // sext(zext(x)) --> zext(x)
01493   if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
01494     return getZeroExtendExpr(SZ->getOperand(), Ty);
01495 
01496   // Before doing any expensive analysis, check to see if we've already
01497   // computed a SCEV for this Op and Ty.
01498   FoldingSetNodeID ID;
01499   ID.AddInteger(scSignExtend);
01500   ID.AddPointer(Op);
01501   ID.AddPointer(Ty);
01502   void *IP = nullptr;
01503   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01504 
01505   // If the input value is provably positive, build a zext instead.
01506   if (isKnownNonNegative(Op))
01507     return getZeroExtendExpr(Op, Ty);
01508 
01509   // sext(trunc(x)) --> sext(x) or x or trunc(x)
01510   if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) {
01511     // It's possible the bits taken off by the truncate were all sign bits. If
01512     // so, we should be able to simplify this further.
01513     const SCEV *X = ST->getOperand();
01514     ConstantRange CR = getSignedRange(X);
01515     unsigned TruncBits = getTypeSizeInBits(ST->getType());
01516     unsigned NewBits = getTypeSizeInBits(Ty);
01517     if (CR.truncate(TruncBits).signExtend(NewBits).contains(
01518             CR.sextOrTrunc(NewBits)))
01519       return getTruncateOrSignExtend(X, Ty);
01520   }
01521 
01522   // sext(C1 + (C2 * x)) --> C1 + sext(C2 * x) if C1 < C2
01523   if (auto SA = dyn_cast<SCEVAddExpr>(Op)) {
01524     if (SA->getNumOperands() == 2) {
01525       auto SC1 = dyn_cast<SCEVConstant>(SA->getOperand(0));
01526       auto SMul = dyn_cast<SCEVMulExpr>(SA->getOperand(1));
01527       if (SMul && SC1) {
01528         if (auto SC2 = dyn_cast<SCEVConstant>(SMul->getOperand(0))) {
01529           const APInt &C1 = SC1->getValue()->getValue();
01530           const APInt &C2 = SC2->getValue()->getValue();
01531           if (C1.isStrictlyPositive() && C2.isStrictlyPositive() &&
01532               C2.ugt(C1) && C2.isPowerOf2())
01533             return getAddExpr(getSignExtendExpr(SC1, Ty),
01534                               getSignExtendExpr(SMul, Ty));
01535         }
01536       }
01537     }
01538   }
01539   // If the input value is a chrec scev, and we can prove that the value
01540   // did not overflow the old, smaller, value, we can sign extend all of the
01541   // operands (often constants).  This allows analysis of something like
01542   // this:  for (signed char X = 0; X < 100; ++X) { int Y = X; }
01543   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op))
01544     if (AR->isAffine()) {
01545       const SCEV *Start = AR->getStart();
01546       const SCEV *Step = AR->getStepRecurrence(*this);
01547       unsigned BitWidth = getTypeSizeInBits(AR->getType());
01548       const Loop *L = AR->getLoop();
01549 
01550       // If we have special knowledge that this addrec won't overflow,
01551       // we don't need to do any further analysis.
01552       if (AR->getNoWrapFlags(SCEV::FlagNSW))
01553         return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this),
01554                              getSignExtendExpr(Step, Ty),
01555                              L, SCEV::FlagNSW);
01556 
01557       // Check whether the backedge-taken count is SCEVCouldNotCompute.
01558       // Note that this serves two purposes: It filters out loops that are
01559       // simply not analyzable, and it covers the case where this code is
01560       // being called from within backedge-taken count analysis, such that
01561       // attempting to ask for the backedge-taken count would likely result
01562       // in infinite recursion. In the later case, the analysis code will
01563       // cope with a conservative value, and it will take care to purge
01564       // that value once it has finished.
01565       const SCEV *MaxBECount = getMaxBackedgeTakenCount(L);
01566       if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
01567         // Manually compute the final value for AR, checking for
01568         // overflow.
01569 
01570         // Check whether the backedge-taken count can be losslessly casted to
01571         // the addrec's type. The count is always unsigned.
01572         const SCEV *CastedMaxBECount =
01573           getTruncateOrZeroExtend(MaxBECount, Start->getType());
01574         const SCEV *RecastedMaxBECount =
01575           getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType());
01576         if (MaxBECount == RecastedMaxBECount) {
01577           Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
01578           // Check whether Start+Step*MaxBECount has no signed overflow.
01579           const SCEV *SMul = getMulExpr(CastedMaxBECount, Step);
01580           const SCEV *SAdd = getSignExtendExpr(getAddExpr(Start, SMul), WideTy);
01581           const SCEV *WideStart = getSignExtendExpr(Start, WideTy);
01582           const SCEV *WideMaxBECount =
01583             getZeroExtendExpr(CastedMaxBECount, WideTy);
01584           const SCEV *OperandExtendedAdd =
01585             getAddExpr(WideStart,
01586                        getMulExpr(WideMaxBECount,
01587                                   getSignExtendExpr(Step, WideTy)));
01588           if (SAdd == OperandExtendedAdd) {
01589             // Cache knowledge of AR NSW, which is propagated to this AddRec.
01590             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW);
01591             // Return the expression with the addrec on the outside.
01592             return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this),
01593                                  getSignExtendExpr(Step, Ty),
01594                                  L, AR->getNoWrapFlags());
01595           }
01596           // Similar to above, only this time treat the step value as unsigned.
01597           // This covers loops that count up with an unsigned step.
01598           OperandExtendedAdd =
01599             getAddExpr(WideStart,
01600                        getMulExpr(WideMaxBECount,
01601                                   getZeroExtendExpr(Step, WideTy)));
01602           if (SAdd == OperandExtendedAdd) {
01603             // Cache knowledge of AR NSW, which is propagated to this AddRec.
01604             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW);
01605             // Return the expression with the addrec on the outside.
01606             return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this),
01607                                  getZeroExtendExpr(Step, Ty),
01608                                  L, AR->getNoWrapFlags());
01609           }
01610         }
01611 
01612         // If the backedge is guarded by a comparison with the pre-inc value
01613         // the addrec is safe. Also, if the entry is guarded by a comparison
01614         // with the start value and the backedge is guarded by a comparison
01615         // with the post-inc value, the addrec is safe.
01616         ICmpInst::Predicate Pred;
01617         const SCEV *OverflowLimit = getOverflowLimitForStep(Step, &Pred, this);
01618         if (OverflowLimit &&
01619             (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) ||
01620              (isLoopEntryGuardedByCond(L, Pred, Start, OverflowLimit) &&
01621               isLoopBackedgeGuardedByCond(L, Pred, AR->getPostIncExpr(*this),
01622                                           OverflowLimit)))) {
01623           // Cache knowledge of AR NSW, then propagate NSW to the wide AddRec.
01624           const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW);
01625           return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this),
01626                                getSignExtendExpr(Step, Ty),
01627                                L, AR->getNoWrapFlags());
01628         }
01629       }
01630       // If Start and Step are constants, check if we can apply this
01631       // transformation:
01632       // sext{C1,+,C2} --> C1 + sext{0,+,C2} if C1 < C2
01633       auto SC1 = dyn_cast<SCEVConstant>(Start);
01634       auto SC2 = dyn_cast<SCEVConstant>(Step);
01635       if (SC1 && SC2) {
01636         const APInt &C1 = SC1->getValue()->getValue();
01637         const APInt &C2 = SC2->getValue()->getValue();
01638         if (C1.isStrictlyPositive() && C2.isStrictlyPositive() && C2.ugt(C1) &&
01639             C2.isPowerOf2()) {
01640           Start = getSignExtendExpr(Start, Ty);
01641           const SCEV *NewAR = getAddRecExpr(getConstant(AR->getType(), 0), Step,
01642                                             L, AR->getNoWrapFlags());
01643           return getAddExpr(Start, getSignExtendExpr(NewAR, Ty));
01644         }
01645       }
01646     }
01647 
01648   // The cast wasn't folded; create an explicit cast node.
01649   // Recompute the insert position, as it may have been invalidated.
01650   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01651   SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
01652                                                    Op, Ty);
01653   UniqueSCEVs.InsertNode(S, IP);
01654   return S;
01655 }
01656 
01657 /// getAnyExtendExpr - Return a SCEV for the given operand extended with
01658 /// unspecified bits out to the given type.
01659 ///
01660 const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op,
01661                                               Type *Ty) {
01662   assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
01663          "This is not an extending conversion!");
01664   assert(isSCEVable(Ty) &&
01665          "This is not a conversion to a SCEVable type!");
01666   Ty = getEffectiveSCEVType(Ty);
01667 
01668   // Sign-extend negative constants.
01669   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
01670     if (SC->getValue()->getValue().isNegative())
01671       return getSignExtendExpr(Op, Ty);
01672 
01673   // Peel off a truncate cast.
01674   if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Op)) {
01675     const SCEV *NewOp = T->getOperand();
01676     if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty))
01677       return getAnyExtendExpr(NewOp, Ty);
01678     return getTruncateOrNoop(NewOp, Ty);
01679   }
01680 
01681   // Next try a zext cast. If the cast is folded, use it.
01682   const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
01683   if (!isa<SCEVZeroExtendExpr>(ZExt))
01684     return ZExt;
01685 
01686   // Next try a sext cast. If the cast is folded, use it.
01687   const SCEV *SExt = getSignExtendExpr(Op, Ty);
01688   if (!isa<SCEVSignExtendExpr>(SExt))
01689     return SExt;
01690 
01691   // Force the cast to be folded into the operands of an addrec.
01692   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) {
01693     SmallVector<const SCEV *, 4> Ops;
01694     for (const SCEV *Op : AR->operands())
01695       Ops.push_back(getAnyExtendExpr(Op, Ty));
01696     return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagNW);
01697   }
01698 
01699   // If the expression is obviously signed, use the sext cast value.
01700   if (isa<SCEVSMaxExpr>(Op))
01701     return SExt;
01702 
01703   // Absent any other information, use the zext cast value.
01704   return ZExt;
01705 }
01706 
01707 /// CollectAddOperandsWithScales - Process the given Ops list, which is
01708 /// a list of operands to be added under the given scale, update the given
01709 /// map. This is a helper function for getAddRecExpr. As an example of
01710 /// what it does, given a sequence of operands that would form an add
01711 /// expression like this:
01712 ///
01713 ///    m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
01714 ///
01715 /// where A and B are constants, update the map with these values:
01716 ///
01717 ///    (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
01718 ///
01719 /// and add 13 + A*B*29 to AccumulatedConstant.
01720 /// This will allow getAddRecExpr to produce this:
01721 ///
01722 ///    13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
01723 ///
01724 /// This form often exposes folding opportunities that are hidden in
01725 /// the original operand list.
01726 ///
01727 /// Return true iff it appears that any interesting folding opportunities
01728 /// may be exposed. This helps getAddRecExpr short-circuit extra work in
01729 /// the common case where no interesting opportunities are present, and
01730 /// is also used as a check to avoid infinite recursion.
01731 ///
01732 static bool
01733 CollectAddOperandsWithScales(DenseMap<const SCEV *, APInt> &M,
01734                              SmallVectorImpl<const SCEV *> &NewOps,
01735                              APInt &AccumulatedConstant,
01736                              const SCEV *const *Ops, size_t NumOperands,
01737                              const APInt &Scale,
01738                              ScalarEvolution &SE) {
01739   bool Interesting = false;
01740 
01741   // Iterate over the add operands. They are sorted, with constants first.
01742   unsigned i = 0;
01743   while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
01744     ++i;
01745     // Pull a buried constant out to the outside.
01746     if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
01747       Interesting = true;
01748     AccumulatedConstant += Scale * C->getValue()->getValue();
01749   }
01750 
01751   // Next comes everything else. We're especially interested in multiplies
01752   // here, but they're in the middle, so just visit the rest with one loop.
01753   for (; i != NumOperands; ++i) {
01754     const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[i]);
01755     if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) {
01756       APInt NewScale =
01757         Scale * cast<SCEVConstant>(Mul->getOperand(0))->getValue()->getValue();
01758       if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) {
01759         // A multiplication of a constant with another add; recurse.
01760         const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1));
01761         Interesting |=
01762           CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
01763                                        Add->op_begin(), Add->getNumOperands(),
01764                                        NewScale, SE);
01765       } else {
01766         // A multiplication of a constant with some other value. Update
01767         // the map.
01768         SmallVector<const SCEV *, 4> MulOps(Mul->op_begin()+1, Mul->op_end());
01769         const SCEV *Key = SE.getMulExpr(MulOps);
01770         std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair =
01771           M.insert(std::make_pair(Key, NewScale));
01772         if (Pair.second) {
01773           NewOps.push_back(Pair.first->first);
01774         } else {
01775           Pair.first->second += NewScale;
01776           // The map already had an entry for this value, which may indicate
01777           // a folding opportunity.
01778           Interesting = true;
01779         }
01780       }
01781     } else {
01782       // An ordinary operand. Update the map.
01783       std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair =
01784         M.insert(std::make_pair(Ops[i], Scale));
01785       if (Pair.second) {
01786         NewOps.push_back(Pair.first->first);
01787       } else {
01788         Pair.first->second += Scale;
01789         // The map already had an entry for this value, which may indicate
01790         // a folding opportunity.
01791         Interesting = true;
01792       }
01793     }
01794   }
01795 
01796   return Interesting;
01797 }
01798 
01799 namespace {
01800   struct APIntCompare {
01801     bool operator()(const APInt &LHS, const APInt &RHS) const {
01802       return LHS.ult(RHS);
01803     }
01804   };
01805 }
01806 
01807 /// getAddExpr - Get a canonical add expression, or something simpler if
01808 /// possible.
01809 const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
01810                                         SCEV::NoWrapFlags Flags) {
01811   assert(!(Flags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
01812          "only nuw or nsw allowed");
01813   assert(!Ops.empty() && "Cannot get empty add!");
01814   if (Ops.size() == 1) return Ops[0];
01815 #ifndef NDEBUG
01816   Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
01817   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
01818     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
01819            "SCEVAddExpr operand types don't match!");
01820 #endif
01821 
01822   // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
01823   // And vice-versa.
01824   int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
01825   SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask);
01826   if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) {
01827     bool All = true;
01828     for (SmallVectorImpl<const SCEV *>::const_iterator I = Ops.begin(),
01829          E = Ops.end(); I != E; ++I)
01830       if (!isKnownNonNegative(*I)) {
01831         All = false;
01832         break;
01833       }
01834     if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask);
01835   }
01836 
01837   // Sort by complexity, this groups all similar expression types together.
01838   GroupByComplexity(Ops, LI);
01839 
01840   // If there are any constants, fold them together.
01841   unsigned Idx = 0;
01842   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
01843     ++Idx;
01844     assert(Idx < Ops.size());
01845     while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
01846       // We found two constants, fold them together!
01847       Ops[0] = getConstant(LHSC->getValue()->getValue() +
01848                            RHSC->getValue()->getValue());
01849       if (Ops.size() == 2) return Ops[0];
01850       Ops.erase(Ops.begin()+1);  // Erase the folded element
01851       LHSC = cast<SCEVConstant>(Ops[0]);
01852     }
01853 
01854     // If we are left with a constant zero being added, strip it off.
01855     if (LHSC->getValue()->isZero()) {
01856       Ops.erase(Ops.begin());
01857       --Idx;
01858     }
01859 
01860     if (Ops.size() == 1) return Ops[0];
01861   }
01862 
01863   // Okay, check to see if the same value occurs in the operand list more than
01864   // once.  If so, merge them together into an multiply expression.  Since we
01865   // sorted the list, these values are required to be adjacent.
01866   Type *Ty = Ops[0]->getType();
01867   bool FoundMatch = false;
01868   for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
01869     if (Ops[i] == Ops[i+1]) {      //  X + Y + Y  -->  X + Y*2
01870       // Scan ahead to count how many equal operands there are.
01871       unsigned Count = 2;
01872       while (i+Count != e && Ops[i+Count] == Ops[i])
01873         ++Count;
01874       // Merge the values into a multiply.
01875       const SCEV *Scale = getConstant(Ty, Count);
01876       const SCEV *Mul = getMulExpr(Scale, Ops[i]);
01877       if (Ops.size() == Count)
01878         return Mul;
01879       Ops[i] = Mul;
01880       Ops.erase(Ops.begin()+i+1, Ops.begin()+i+Count);
01881       --i; e -= Count - 1;
01882       FoundMatch = true;
01883     }
01884   if (FoundMatch)
01885     return getAddExpr(Ops, Flags);
01886 
01887   // Check for truncates. If all the operands are truncated from the same
01888   // type, see if factoring out the truncate would permit the result to be
01889   // folded. eg., trunc(x) + m*trunc(n) --> trunc(x + trunc(m)*n)
01890   // if the contents of the resulting outer trunc fold to something simple.
01891   for (; Idx < Ops.size() && isa<SCEVTruncateExpr>(Ops[Idx]); ++Idx) {
01892     const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Ops[Idx]);
01893     Type *DstType = Trunc->getType();
01894     Type *SrcType = Trunc->getOperand()->getType();
01895     SmallVector<const SCEV *, 8> LargeOps;
01896     bool Ok = true;
01897     // Check all the operands to see if they can be represented in the
01898     // source type of the truncate.
01899     for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
01900       if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Ops[i])) {
01901         if (T->getOperand()->getType() != SrcType) {
01902           Ok = false;
01903           break;
01904         }
01905         LargeOps.push_back(T->getOperand());
01906       } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
01907         LargeOps.push_back(getAnyExtendExpr(C, SrcType));
01908       } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Ops[i])) {
01909         SmallVector<const SCEV *, 8> LargeMulOps;
01910         for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
01911           if (const SCEVTruncateExpr *T =
01912                 dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) {
01913             if (T->getOperand()->getType() != SrcType) {
01914               Ok = false;
01915               break;
01916             }
01917             LargeMulOps.push_back(T->getOperand());
01918           } else if (const SCEVConstant *C =
01919                        dyn_cast<SCEVConstant>(M->getOperand(j))) {
01920             LargeMulOps.push_back(getAnyExtendExpr(C, SrcType));
01921           } else {
01922             Ok = false;
01923             break;
01924           }
01925         }
01926         if (Ok)
01927           LargeOps.push_back(getMulExpr(LargeMulOps));
01928       } else {
01929         Ok = false;
01930         break;
01931       }
01932     }
01933     if (Ok) {
01934       // Evaluate the expression in the larger type.
01935       const SCEV *Fold = getAddExpr(LargeOps, Flags);
01936       // If it folds to something simple, use it. Otherwise, don't.
01937       if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold))
01938         return getTruncateExpr(Fold, DstType);
01939     }
01940   }
01941 
01942   // Skip past any other cast SCEVs.
01943   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
01944     ++Idx;
01945 
01946   // If there are add operands they would be next.
01947   if (Idx < Ops.size()) {
01948     bool DeletedAdd = false;
01949     while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) {
01950       // If we have an add, expand the add operands onto the end of the operands
01951       // list.
01952       Ops.erase(Ops.begin()+Idx);
01953       Ops.append(Add->op_begin(), Add->op_end());
01954       DeletedAdd = true;
01955     }
01956 
01957     // If we deleted at least one add, we added operands to the end of the list,
01958     // and they are not necessarily sorted.  Recurse to resort and resimplify
01959     // any operands we just acquired.
01960     if (DeletedAdd)
01961       return getAddExpr(Ops);
01962   }
01963 
01964   // Skip over the add expression until we get to a multiply.
01965   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
01966     ++Idx;
01967 
01968   // Check to see if there are any folding opportunities present with
01969   // operands multiplied by constant values.
01970   if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) {
01971     uint64_t BitWidth = getTypeSizeInBits(Ty);
01972     DenseMap<const SCEV *, APInt> M;
01973     SmallVector<const SCEV *, 8> NewOps;
01974     APInt AccumulatedConstant(BitWidth, 0);
01975     if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
01976                                      Ops.data(), Ops.size(),
01977                                      APInt(BitWidth, 1), *this)) {
01978       // Some interesting folding opportunity is present, so its worthwhile to
01979       // re-generate the operands list. Group the operands by constant scale,
01980       // to avoid multiplying by the same constant scale multiple times.
01981       std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists;
01982       for (SmallVectorImpl<const SCEV *>::const_iterator I = NewOps.begin(),
01983            E = NewOps.end(); I != E; ++I)
01984         MulOpLists[M.find(*I)->second].push_back(*I);
01985       // Re-generate the operands list.
01986       Ops.clear();
01987       if (AccumulatedConstant != 0)
01988         Ops.push_back(getConstant(AccumulatedConstant));
01989       for (std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare>::iterator
01990            I = MulOpLists.begin(), E = MulOpLists.end(); I != E; ++I)
01991         if (I->first != 0)
01992           Ops.push_back(getMulExpr(getConstant(I->first),
01993                                    getAddExpr(I->second)));
01994       if (Ops.empty())
01995         return getConstant(Ty, 0);
01996       if (Ops.size() == 1)
01997         return Ops[0];
01998       return getAddExpr(Ops);
01999     }
02000   }
02001 
02002   // If we are adding something to a multiply expression, make sure the
02003   // something is not already an operand of the multiply.  If so, merge it into
02004   // the multiply.
02005   for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) {
02006     const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]);
02007     for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
02008       const SCEV *MulOpSCEV = Mul->getOperand(MulOp);
02009       if (isa<SCEVConstant>(MulOpSCEV))
02010         continue;
02011       for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp)
02012         if (MulOpSCEV == Ops[AddOp]) {
02013           // Fold W + X + (X * Y * Z)  -->  W + (X * ((Y*Z)+1))
02014           const SCEV *InnerMul = Mul->getOperand(MulOp == 0);
02015           if (Mul->getNumOperands() != 2) {
02016             // If the multiply has more than two operands, we must get the
02017             // Y*Z term.
02018             SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(),
02019                                                 Mul->op_begin()+MulOp);
02020             MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end());
02021             InnerMul = getMulExpr(MulOps);
02022           }
02023           const SCEV *One = getConstant(Ty, 1);
02024           const SCEV *AddOne = getAddExpr(One, InnerMul);
02025           const SCEV *OuterMul = getMulExpr(AddOne, MulOpSCEV);
02026           if (Ops.size() == 2) return OuterMul;
02027           if (AddOp < Idx) {
02028             Ops.erase(Ops.begin()+AddOp);
02029             Ops.erase(Ops.begin()+Idx-1);
02030           } else {
02031             Ops.erase(Ops.begin()+Idx);
02032             Ops.erase(Ops.begin()+AddOp-1);
02033           }
02034           Ops.push_back(OuterMul);
02035           return getAddExpr(Ops);
02036         }
02037 
02038       // Check this multiply against other multiplies being added together.
02039       for (unsigned OtherMulIdx = Idx+1;
02040            OtherMulIdx < Ops.size() && isa<SCEVMulExpr>(Ops[OtherMulIdx]);
02041            ++OtherMulIdx) {
02042         const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[OtherMulIdx]);
02043         // If MulOp occurs in OtherMul, we can fold the two multiplies
02044         // together.
02045         for (unsigned OMulOp = 0, e = OtherMul->getNumOperands();
02046              OMulOp != e; ++OMulOp)
02047           if (OtherMul->getOperand(OMulOp) == MulOpSCEV) {
02048             // Fold X + (A*B*C) + (A*D*E) --> X + (A*(B*C+D*E))
02049             const SCEV *InnerMul1 = Mul->getOperand(MulOp == 0);
02050             if (Mul->getNumOperands() != 2) {
02051               SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(),
02052                                                   Mul->op_begin()+MulOp);
02053               MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end());
02054               InnerMul1 = getMulExpr(MulOps);
02055             }
02056             const SCEV *InnerMul2 = OtherMul->getOperand(OMulOp == 0);
02057             if (OtherMul->getNumOperands() != 2) {
02058               SmallVector<const SCEV *, 4> MulOps(OtherMul->op_begin(),
02059                                                   OtherMul->op_begin()+OMulOp);
02060               MulOps.append(OtherMul->op_begin()+OMulOp+1, OtherMul->op_end());
02061               InnerMul2 = getMulExpr(MulOps);
02062             }
02063             const SCEV *InnerMulSum = getAddExpr(InnerMul1,InnerMul2);
02064             const SCEV *OuterMul = getMulExpr(MulOpSCEV, InnerMulSum);
02065             if (Ops.size() == 2) return OuterMul;
02066             Ops.erase(Ops.begin()+Idx);
02067             Ops.erase(Ops.begin()+OtherMulIdx-1);
02068             Ops.push_back(OuterMul);
02069             return getAddExpr(Ops);
02070           }
02071       }
02072     }
02073   }
02074 
02075   // If there are any add recurrences in the operands list, see if any other
02076   // added values are loop invariant.  If so, we can fold them into the
02077   // recurrence.
02078   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
02079     ++Idx;
02080 
02081   // Scan over all recurrences, trying to fold loop invariants into them.
02082   for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
02083     // Scan all of the other operands to this add and add them to the vector if
02084     // they are loop invariant w.r.t. the recurrence.
02085     SmallVector<const SCEV *, 8> LIOps;
02086     const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
02087     const Loop *AddRecLoop = AddRec->getLoop();
02088     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02089       if (isLoopInvariant(Ops[i], AddRecLoop)) {
02090         LIOps.push_back(Ops[i]);
02091         Ops.erase(Ops.begin()+i);
02092         --i; --e;
02093       }
02094 
02095     // If we found some loop invariants, fold them into the recurrence.
02096     if (!LIOps.empty()) {
02097       //  NLI + LI + {Start,+,Step}  -->  NLI + {LI+Start,+,Step}
02098       LIOps.push_back(AddRec->getStart());
02099 
02100       SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(),
02101                                              AddRec->op_end());
02102       AddRecOps[0] = getAddExpr(LIOps);
02103 
02104       // Build the new addrec. Propagate the NUW and NSW flags if both the
02105       // outer add and the inner addrec are guaranteed to have no overflow.
02106       // Always propagate NW.
02107       Flags = AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW));
02108       const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags);
02109 
02110       // If all of the other operands were loop invariant, we are done.
02111       if (Ops.size() == 1) return NewRec;
02112 
02113       // Otherwise, add the folded AddRec by the non-invariant parts.
02114       for (unsigned i = 0;; ++i)
02115         if (Ops[i] == AddRec) {
02116           Ops[i] = NewRec;
02117           break;
02118         }
02119       return getAddExpr(Ops);
02120     }
02121 
02122     // Okay, if there weren't any loop invariants to be folded, check to see if
02123     // there are multiple AddRec's with the same loop induction variable being
02124     // added together.  If so, we can fold them.
02125     for (unsigned OtherIdx = Idx+1;
02126          OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
02127          ++OtherIdx)
02128       if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) {
02129         // Other + {A,+,B}<L> + {C,+,D}<L>  -->  Other + {A+C,+,B+D}<L>
02130         SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(),
02131                                                AddRec->op_end());
02132         for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
02133              ++OtherIdx)
02134           if (const SCEVAddRecExpr *OtherAddRec =
02135                 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]))
02136             if (OtherAddRec->getLoop() == AddRecLoop) {
02137               for (unsigned i = 0, e = OtherAddRec->getNumOperands();
02138                    i != e; ++i) {
02139                 if (i >= AddRecOps.size()) {
02140                   AddRecOps.append(OtherAddRec->op_begin()+i,
02141                                    OtherAddRec->op_end());
02142                   break;
02143                 }
02144                 AddRecOps[i] = getAddExpr(AddRecOps[i],
02145                                           OtherAddRec->getOperand(i));
02146               }
02147               Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
02148             }
02149         // Step size has changed, so we cannot guarantee no self-wraparound.
02150         Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap);
02151         return getAddExpr(Ops);
02152       }
02153 
02154     // Otherwise couldn't fold anything into this recurrence.  Move onto the
02155     // next one.
02156   }
02157 
02158   // Okay, it looks like we really DO need an add expr.  Check to see if we
02159   // already have one, otherwise create a new one.
02160   FoldingSetNodeID ID;
02161   ID.AddInteger(scAddExpr);
02162   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02163     ID.AddPointer(Ops[i]);
02164   void *IP = nullptr;
02165   SCEVAddExpr *S =
02166     static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
02167   if (!S) {
02168     const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
02169     std::uninitialized_copy(Ops.begin(), Ops.end(), O);
02170     S = new (SCEVAllocator) SCEVAddExpr(ID.Intern(SCEVAllocator),
02171                                         O, Ops.size());
02172     UniqueSCEVs.InsertNode(S, IP);
02173   }
02174   S->setNoWrapFlags(Flags);
02175   return S;
02176 }
02177 
02178 static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
02179   uint64_t k = i*j;
02180   if (j > 1 && k / j != i) Overflow = true;
02181   return k;
02182 }
02183 
02184 /// Compute the result of "n choose k", the binomial coefficient.  If an
02185 /// intermediate computation overflows, Overflow will be set and the return will
02186 /// be garbage. Overflow is not cleared on absence of overflow.
02187 static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
02188   // We use the multiplicative formula:
02189   //     n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
02190   // At each iteration, we take the n-th term of the numeral and divide by the
02191   // (k-n)th term of the denominator.  This division will always produce an
02192   // integral result, and helps reduce the chance of overflow in the
02193   // intermediate computations. However, we can still overflow even when the
02194   // final result would fit.
02195 
02196   if (n == 0 || n == k) return 1;
02197   if (k > n) return 0;
02198 
02199   if (k > n/2)
02200     k = n-k;
02201 
02202   uint64_t r = 1;
02203   for (uint64_t i = 1; i <= k; ++i) {
02204     r = umul_ov(r, n-(i-1), Overflow);
02205     r /= i;
02206   }
02207   return r;
02208 }
02209 
02210 /// getMulExpr - Get a canonical multiply expression, or something simpler if
02211 /// possible.
02212 const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
02213                                         SCEV::NoWrapFlags Flags) {
02214   assert(Flags == maskFlags(Flags, SCEV::FlagNUW | SCEV::FlagNSW) &&
02215          "only nuw or nsw allowed");
02216   assert(!Ops.empty() && "Cannot get empty mul!");
02217   if (Ops.size() == 1) return Ops[0];
02218 #ifndef NDEBUG
02219   Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
02220   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
02221     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
02222            "SCEVMulExpr operand types don't match!");
02223 #endif
02224 
02225   // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
02226   // And vice-versa.
02227   int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
02228   SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask);
02229   if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) {
02230     bool All = true;
02231     for (SmallVectorImpl<const SCEV *>::const_iterator I = Ops.begin(),
02232          E = Ops.end(); I != E; ++I)
02233       if (!isKnownNonNegative(*I)) {
02234         All = false;
02235         break;
02236       }
02237     if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask);
02238   }
02239 
02240   // Sort by complexity, this groups all similar expression types together.
02241   GroupByComplexity(Ops, LI);
02242 
02243   // If there are any constants, fold them together.
02244   unsigned Idx = 0;
02245   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
02246 
02247     // C1*(C2+V) -> C1*C2 + C1*V
02248     if (Ops.size() == 2)
02249       if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1]))
02250         if (Add->getNumOperands() == 2 &&
02251             isa<SCEVConstant>(Add->getOperand(0)))
02252           return getAddExpr(getMulExpr(LHSC, Add->getOperand(0)),
02253                             getMulExpr(LHSC, Add->getOperand(1)));
02254 
02255     ++Idx;
02256     while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
02257       // We found two constants, fold them together!
02258       ConstantInt *Fold = ConstantInt::get(getContext(),
02259                                            LHSC->getValue()->getValue() *
02260                                            RHSC->getValue()->getValue());
02261       Ops[0] = getConstant(Fold);
02262       Ops.erase(Ops.begin()+1);  // Erase the folded element
02263       if (Ops.size() == 1) return Ops[0];
02264       LHSC = cast<SCEVConstant>(Ops[0]);
02265     }
02266 
02267     // If we are left with a constant one being multiplied, strip it off.
02268     if (cast<SCEVConstant>(Ops[0])->getValue()->equalsInt(1)) {
02269       Ops.erase(Ops.begin());
02270       --Idx;
02271     } else if (cast<SCEVConstant>(Ops[0])->getValue()->isZero()) {
02272       // If we have a multiply of zero, it will always be zero.
02273       return Ops[0];
02274     } else if (Ops[0]->isAllOnesValue()) {
02275       // If we have a mul by -1 of an add, try distributing the -1 among the
02276       // add operands.
02277       if (Ops.size() == 2) {
02278         if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) {
02279           SmallVector<const SCEV *, 4> NewOps;
02280           bool AnyFolded = false;
02281           for (SCEVAddRecExpr::op_iterator I = Add->op_begin(),
02282                  E = Add->op_end(); I != E; ++I) {
02283             const SCEV *Mul = getMulExpr(Ops[0], *I);
02284             if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true;
02285             NewOps.push_back(Mul);
02286           }
02287           if (AnyFolded)
02288             return getAddExpr(NewOps);
02289         }
02290         else if (const SCEVAddRecExpr *
02291                  AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) {
02292           // Negation preserves a recurrence's no self-wrap property.
02293           SmallVector<const SCEV *, 4> Operands;
02294           for (SCEVAddRecExpr::op_iterator I = AddRec->op_begin(),
02295                  E = AddRec->op_end(); I != E; ++I) {
02296             Operands.push_back(getMulExpr(Ops[0], *I));
02297           }
02298           return getAddRecExpr(Operands, AddRec->getLoop(),
02299                                AddRec->getNoWrapFlags(SCEV::FlagNW));
02300         }
02301       }
02302     }
02303 
02304     if (Ops.size() == 1)
02305       return Ops[0];
02306   }
02307 
02308   // Skip over the add expression until we get to a multiply.
02309   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
02310     ++Idx;
02311 
02312   // If there are mul operands inline them all into this expression.
02313   if (Idx < Ops.size()) {
02314     bool DeletedMul = false;
02315     while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
02316       // If we have an mul, expand the mul operands onto the end of the operands
02317       // list.
02318       Ops.erase(Ops.begin()+Idx);
02319       Ops.append(Mul->op_begin(), Mul->op_end());
02320       DeletedMul = true;
02321     }
02322 
02323     // If we deleted at least one mul, we added operands to the end of the list,
02324     // and they are not necessarily sorted.  Recurse to resort and resimplify
02325     // any operands we just acquired.
02326     if (DeletedMul)
02327       return getMulExpr(Ops);
02328   }
02329 
02330   // If there are any add recurrences in the operands list, see if any other
02331   // added values are loop invariant.  If so, we can fold them into the
02332   // recurrence.
02333   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
02334     ++Idx;
02335 
02336   // Scan over all recurrences, trying to fold loop invariants into them.
02337   for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
02338     // Scan all of the other operands to this mul and add them to the vector if
02339     // they are loop invariant w.r.t. the recurrence.
02340     SmallVector<const SCEV *, 8> LIOps;
02341     const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
02342     const Loop *AddRecLoop = AddRec->getLoop();
02343     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02344       if (isLoopInvariant(Ops[i], AddRecLoop)) {
02345         LIOps.push_back(Ops[i]);
02346         Ops.erase(Ops.begin()+i);
02347         --i; --e;
02348       }
02349 
02350     // If we found some loop invariants, fold them into the recurrence.
02351     if (!LIOps.empty()) {
02352       //  NLI * LI * {Start,+,Step}  -->  NLI * {LI*Start,+,LI*Step}
02353       SmallVector<const SCEV *, 4> NewOps;
02354       NewOps.reserve(AddRec->getNumOperands());
02355       const SCEV *Scale = getMulExpr(LIOps);
02356       for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i)
02357         NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i)));
02358 
02359       // Build the new addrec. Propagate the NUW and NSW flags if both the
02360       // outer mul and the inner addrec are guaranteed to have no overflow.
02361       //
02362       // No self-wrap cannot be guaranteed after changing the step size, but
02363       // will be inferred if either NUW or NSW is true.
02364       Flags = AddRec->getNoWrapFlags(clearFlags(Flags, SCEV::FlagNW));
02365       const SCEV *NewRec = getAddRecExpr(NewOps, AddRecLoop, Flags);
02366 
02367       // If all of the other operands were loop invariant, we are done.
02368       if (Ops.size() == 1) return NewRec;
02369 
02370       // Otherwise, multiply the folded AddRec by the non-invariant parts.
02371       for (unsigned i = 0;; ++i)
02372         if (Ops[i] == AddRec) {
02373           Ops[i] = NewRec;
02374           break;
02375         }
02376       return getMulExpr(Ops);
02377     }
02378 
02379     // Okay, if there weren't any loop invariants to be folded, check to see if
02380     // there are multiple AddRec's with the same loop induction variable being
02381     // multiplied together.  If so, we can fold them.
02382 
02383     // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
02384     // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
02385     //       choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
02386     //   ]]],+,...up to x=2n}.
02387     // Note that the arguments to choose() are always integers with values
02388     // known at compile time, never SCEV objects.
02389     //
02390     // The implementation avoids pointless extra computations when the two
02391     // addrec's are of different length (mathematically, it's equivalent to
02392     // an infinite stream of zeros on the right).
02393     bool OpsModified = false;
02394     for (unsigned OtherIdx = Idx+1;
02395          OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
02396          ++OtherIdx) {
02397       const SCEVAddRecExpr *OtherAddRec =
02398         dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]);
02399       if (!OtherAddRec || OtherAddRec->getLoop() != AddRecLoop)
02400         continue;
02401 
02402       bool Overflow = false;
02403       Type *Ty = AddRec->getType();
02404       bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
02405       SmallVector<const SCEV*, 7> AddRecOps;
02406       for (int x = 0, xe = AddRec->getNumOperands() +
02407              OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
02408         const SCEV *Term = getConstant(Ty, 0);
02409         for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
02410           uint64_t Coeff1 = Choose(x, 2*x - y, Overflow);
02411           for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1),
02412                  ze = std::min(x+1, (int)OtherAddRec->getNumOperands());
02413                z < ze && !Overflow; ++z) {
02414             uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow);
02415             uint64_t Coeff;
02416             if (LargerThan64Bits)
02417               Coeff = umul_ov(Coeff1, Coeff2, Overflow);
02418             else
02419               Coeff = Coeff1*Coeff2;
02420             const SCEV *CoeffTerm = getConstant(Ty, Coeff);
02421             const SCEV *Term1 = AddRec->getOperand(y-z);
02422             const SCEV *Term2 = OtherAddRec->getOperand(z);
02423             Term = getAddExpr(Term, getMulExpr(CoeffTerm, Term1,Term2));
02424           }
02425         }
02426         AddRecOps.push_back(Term);
02427       }
02428       if (!Overflow) {
02429         const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRec->getLoop(),
02430                                               SCEV::FlagAnyWrap);
02431         if (Ops.size() == 2) return NewAddRec;
02432         Ops[Idx] = NewAddRec;
02433         Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
02434         OpsModified = true;
02435         AddRec = dyn_cast<SCEVAddRecExpr>(NewAddRec);
02436         if (!AddRec)
02437           break;
02438       }
02439     }
02440     if (OpsModified)
02441       return getMulExpr(Ops);
02442 
02443     // Otherwise couldn't fold anything into this recurrence.  Move onto the
02444     // next one.
02445   }
02446 
02447   // Okay, it looks like we really DO need an mul expr.  Check to see if we
02448   // already have one, otherwise create a new one.
02449   FoldingSetNodeID ID;
02450   ID.AddInteger(scMulExpr);
02451   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02452     ID.AddPointer(Ops[i]);
02453   void *IP = nullptr;
02454   SCEVMulExpr *S =
02455     static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
02456   if (!S) {
02457     const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
02458     std::uninitialized_copy(Ops.begin(), Ops.end(), O);
02459     S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator),
02460                                         O, Ops.size());
02461     UniqueSCEVs.InsertNode(S, IP);
02462   }
02463   S->setNoWrapFlags(Flags);
02464   return S;
02465 }
02466 
02467 /// getUDivExpr - Get a canonical unsigned division expression, or something
02468 /// simpler if possible.
02469 const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS,
02470                                          const SCEV *RHS) {
02471   assert(getEffectiveSCEVType(LHS->getType()) ==
02472          getEffectiveSCEVType(RHS->getType()) &&
02473          "SCEVUDivExpr operand types don't match!");
02474 
02475   if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
02476     if (RHSC->getValue()->equalsInt(1))
02477       return LHS;                               // X udiv 1 --> x
02478     // If the denominator is zero, the result of the udiv is undefined. Don't
02479     // try to analyze it, because the resolution chosen here may differ from
02480     // the resolution chosen in other parts of the compiler.
02481     if (!RHSC->getValue()->isZero()) {
02482       // Determine if the division can be folded into the operands of
02483       // its operands.
02484       // TODO: Generalize this to non-constants by using known-bits information.
02485       Type *Ty = LHS->getType();
02486       unsigned LZ = RHSC->getValue()->getValue().countLeadingZeros();
02487       unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
02488       // For non-power-of-two values, effectively round the value up to the
02489       // nearest power of two.
02490       if (!RHSC->getValue()->getValue().isPowerOf2())
02491         ++MaxShiftAmt;
02492       IntegerType *ExtTy =
02493         IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt);
02494       if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
02495         if (const SCEVConstant *Step =
02496             dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) {
02497           // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
02498           const APInt &StepInt = Step->getValue()->getValue();
02499           const APInt &DivInt = RHSC->getValue()->getValue();
02500           if (!StepInt.urem(DivInt) &&
02501               getZeroExtendExpr(AR, ExtTy) ==
02502               getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
02503                             getZeroExtendExpr(Step, ExtTy),
02504                             AR->getLoop(), SCEV::FlagAnyWrap)) {
02505             SmallVector<const SCEV *, 4> Operands;
02506             for (unsigned i = 0, e = AR->getNumOperands(); i != e; ++i)
02507               Operands.push_back(getUDivExpr(AR->getOperand(i), RHS));
02508             return getAddRecExpr(Operands, AR->getLoop(),
02509                                  SCEV::FlagNW);
02510           }
02511           /// Get a canonical UDivExpr for a recurrence.
02512           /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
02513           // We can currently only fold X%N if X is constant.
02514           const SCEVConstant *StartC = dyn_cast<SCEVConstant>(AR->getStart());
02515           if (StartC && !DivInt.urem(StepInt) &&
02516               getZeroExtendExpr(AR, ExtTy) ==
02517               getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
02518                             getZeroExtendExpr(Step, ExtTy),
02519                             AR->getLoop(), SCEV::FlagAnyWrap)) {
02520             const APInt &StartInt = StartC->getValue()->getValue();
02521             const APInt &StartRem = StartInt.urem(StepInt);
02522             if (StartRem != 0)
02523               LHS = getAddRecExpr(getConstant(StartInt - StartRem), Step,
02524                                   AR->getLoop(), SCEV::FlagNW);
02525           }
02526         }
02527       // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
02528       if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) {
02529         SmallVector<const SCEV *, 4> Operands;
02530         for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i)
02531           Operands.push_back(getZeroExtendExpr(M->getOperand(i), ExtTy));
02532         if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands))
02533           // Find an operand that's safely divisible.
02534           for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
02535             const SCEV *Op = M->getOperand(i);
02536             const SCEV *Div = getUDivExpr(Op, RHSC);
02537             if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) {
02538               Operands = SmallVector<const SCEV *, 4>(M->op_begin(),
02539                                                       M->op_end());
02540               Operands[i] = Div;
02541               return getMulExpr(Operands);
02542             }
02543           }
02544       }
02545       // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded.
02546       if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) {
02547         SmallVector<const SCEV *, 4> Operands;
02548         for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i)
02549           Operands.push_back(getZeroExtendExpr(A->getOperand(i), ExtTy));
02550         if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) {
02551           Operands.clear();
02552           for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
02553             const SCEV *Op = getUDivExpr(A->getOperand(i), RHS);
02554             if (isa<SCEVUDivExpr>(Op) ||
02555                 getMulExpr(Op, RHS) != A->getOperand(i))
02556               break;
02557             Operands.push_back(Op);
02558           }
02559           if (Operands.size() == A->getNumOperands())
02560             return getAddExpr(Operands);
02561         }
02562       }
02563 
02564       // Fold if both operands are constant.
02565       if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
02566         Constant *LHSCV = LHSC->getValue();
02567         Constant *RHSCV = RHSC->getValue();
02568         return getConstant(cast<ConstantInt>(ConstantExpr::getUDiv(LHSCV,
02569                                                                    RHSCV)));
02570       }
02571     }
02572   }
02573 
02574   FoldingSetNodeID ID;
02575   ID.AddInteger(scUDivExpr);
02576   ID.AddPointer(LHS);
02577   ID.AddPointer(RHS);
02578   void *IP = nullptr;
02579   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
02580   SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator),
02581                                              LHS, RHS);
02582   UniqueSCEVs.InsertNode(S, IP);
02583   return S;
02584 }
02585 
02586 static const APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) {
02587   APInt A = C1->getValue()->getValue().abs();
02588   APInt B = C2->getValue()->getValue().abs();
02589   uint32_t ABW = A.getBitWidth();
02590   uint32_t BBW = B.getBitWidth();
02591 
02592   if (ABW > BBW)
02593     B = B.zext(ABW);
02594   else if (ABW < BBW)
02595     A = A.zext(BBW);
02596 
02597   return APIntOps::GreatestCommonDivisor(A, B);
02598 }
02599 
02600 /// getUDivExactExpr - Get a canonical unsigned division expression, or
02601 /// something simpler if possible. There is no representation for an exact udiv
02602 /// in SCEV IR, but we can attempt to remove factors from the LHS and RHS.
02603 /// We can't do this when it's not exact because the udiv may be clearing bits.
02604 const SCEV *ScalarEvolution::getUDivExactExpr(const SCEV *LHS,
02605                                               const SCEV *RHS) {
02606   // TODO: we could try to find factors in all sorts of things, but for now we
02607   // just deal with u/exact (multiply, constant). See SCEVDivision towards the
02608   // end of this file for inspiration.
02609 
02610   const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS);
02611   if (!Mul)
02612     return getUDivExpr(LHS, RHS);
02613 
02614   if (const SCEVConstant *RHSCst = dyn_cast<SCEVConstant>(RHS)) {
02615     // If the mulexpr multiplies by a constant, then that constant must be the
02616     // first element of the mulexpr.
02617     if (const SCEVConstant *LHSCst =
02618             dyn_cast<SCEVConstant>(Mul->getOperand(0))) {
02619       if (LHSCst == RHSCst) {
02620         SmallVector<const SCEV *, 2> Operands;
02621         Operands.append(Mul->op_begin() + 1, Mul->op_end());
02622         return getMulExpr(Operands);
02623       }
02624 
02625       // We can't just assume that LHSCst divides RHSCst cleanly, it could be
02626       // that there's a factor provided by one of the other terms. We need to
02627       // check.
02628       APInt Factor = gcd(LHSCst, RHSCst);
02629       if (!Factor.isIntN(1)) {
02630         LHSCst = cast<SCEVConstant>(
02631             getConstant(LHSCst->getValue()->getValue().udiv(Factor)));
02632         RHSCst = cast<SCEVConstant>(
02633             getConstant(RHSCst->getValue()->getValue().udiv(Factor)));
02634         SmallVector<const SCEV *, 2> Operands;
02635         Operands.push_back(LHSCst);
02636         Operands.append(Mul->op_begin() + 1, Mul->op_end());
02637         LHS = getMulExpr(Operands);
02638         RHS = RHSCst;
02639         Mul = dyn_cast<SCEVMulExpr>(LHS);
02640         if (!Mul)
02641           return getUDivExactExpr(LHS, RHS);
02642       }
02643     }
02644   }
02645 
02646   for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
02647     if (Mul->getOperand(i) == RHS) {
02648       SmallVector<const SCEV *, 2> Operands;
02649       Operands.append(Mul->op_begin(), Mul->op_begin() + i);
02650       Operands.append(Mul->op_begin() + i + 1, Mul->op_end());
02651       return getMulExpr(Operands);
02652     }
02653   }
02654 
02655   return getUDivExpr(LHS, RHS);
02656 }
02657 
02658 /// getAddRecExpr - Get an add recurrence expression for the specified loop.
02659 /// Simplify the expression as much as possible.
02660 const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, const SCEV *Step,
02661                                            const Loop *L,
02662                                            SCEV::NoWrapFlags Flags) {
02663   SmallVector<const SCEV *, 4> Operands;
02664   Operands.push_back(Start);
02665   if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step))
02666     if (StepChrec->getLoop() == L) {
02667       Operands.append(StepChrec->op_begin(), StepChrec->op_end());
02668       return getAddRecExpr(Operands, L, maskFlags(Flags, SCEV::FlagNW));
02669     }
02670 
02671   Operands.push_back(Step);
02672   return getAddRecExpr(Operands, L, Flags);
02673 }
02674 
02675 /// getAddRecExpr - Get an add recurrence expression for the specified loop.
02676 /// Simplify the expression as much as possible.
02677 const SCEV *
02678 ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands,
02679                                const Loop *L, SCEV::NoWrapFlags Flags) {
02680   if (Operands.size() == 1) return Operands[0];
02681 #ifndef NDEBUG
02682   Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
02683   for (unsigned i = 1, e = Operands.size(); i != e; ++i)
02684     assert(getEffectiveSCEVType(Operands[i]->getType()) == ETy &&
02685            "SCEVAddRecExpr operand types don't match!");
02686   for (unsigned i = 0, e = Operands.size(); i != e; ++i)
02687     assert(isLoopInvariant(Operands[i], L) &&
02688            "SCEVAddRecExpr operand is not loop-invariant!");
02689 #endif
02690 
02691   if (Operands.back()->isZero()) {
02692     Operands.pop_back();
02693     return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); // {X,+,0}  -->  X
02694   }
02695 
02696   // It's tempting to want to call getMaxBackedgeTakenCount count here and
02697   // use that information to infer NUW and NSW flags. However, computing a
02698   // BE count requires calling getAddRecExpr, so we may not yet have a
02699   // meaningful BE count at this point (and if we don't, we'd be stuck
02700   // with a SCEVCouldNotCompute as the cached BE count).
02701 
02702   // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
02703   // And vice-versa.
02704   int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
02705   SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask);
02706   if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) {
02707     bool All = true;
02708     for (SmallVectorImpl<const SCEV *>::const_iterator I = Operands.begin(),
02709          E = Operands.end(); I != E; ++I)
02710       if (!isKnownNonNegative(*I)) {
02711         All = false;
02712         break;
02713       }
02714     if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask);
02715   }
02716 
02717   // Canonicalize nested AddRecs in by nesting them in order of loop depth.
02718   if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) {
02719     const Loop *NestedLoop = NestedAR->getLoop();
02720     if (L->contains(NestedLoop) ?
02721         (L->getLoopDepth() < NestedLoop->getLoopDepth()) :
02722         (!NestedLoop->contains(L) &&
02723          DT->dominates(L->getHeader(), NestedLoop->getHeader()))) {
02724       SmallVector<const SCEV *, 4> NestedOperands(NestedAR->op_begin(),
02725                                                   NestedAR->op_end());
02726       Operands[0] = NestedAR->getStart();
02727       // AddRecs require their operands be loop-invariant with respect to their
02728       // loops. Don't perform this transformation if it would break this
02729       // requirement.
02730       bool AllInvariant = true;
02731       for (unsigned i = 0, e = Operands.size(); i != e; ++i)
02732         if (!isLoopInvariant(Operands[i], L)) {
02733           AllInvariant = false;
02734           break;
02735         }
02736       if (AllInvariant) {
02737         // Create a recurrence for the outer loop with the same step size.
02738         //
02739         // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the
02740         // inner recurrence has the same property.
02741         SCEV::NoWrapFlags OuterFlags =
02742           maskFlags(Flags, SCEV::FlagNW | NestedAR->getNoWrapFlags());
02743 
02744         NestedOperands[0] = getAddRecExpr(Operands, L, OuterFlags);
02745         AllInvariant = true;
02746         for (unsigned i = 0, e = NestedOperands.size(); i != e; ++i)
02747           if (!isLoopInvariant(NestedOperands[i], NestedLoop)) {
02748             AllInvariant = false;
02749             break;
02750           }
02751         if (AllInvariant) {
02752           // Ok, both add recurrences are valid after the transformation.
02753           //
02754           // The inner recurrence keeps its NW flag but only keeps NUW/NSW if
02755           // the outer recurrence has the same property.
02756           SCEV::NoWrapFlags InnerFlags =
02757             maskFlags(NestedAR->getNoWrapFlags(), SCEV::FlagNW | Flags);
02758           return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags);
02759         }
02760       }
02761       // Reset Operands to its original state.
02762       Operands[0] = NestedAR;
02763     }
02764   }
02765 
02766   // Okay, it looks like we really DO need an addrec expr.  Check to see if we
02767   // already have one, otherwise create a new one.
02768   FoldingSetNodeID ID;
02769   ID.AddInteger(scAddRecExpr);
02770   for (unsigned i = 0, e = Operands.size(); i != e; ++i)
02771     ID.AddPointer(Operands[i]);
02772   ID.AddPointer(L);
02773   void *IP = nullptr;
02774   SCEVAddRecExpr *S =
02775     static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
02776   if (!S) {
02777     const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Operands.size());
02778     std::uninitialized_copy(Operands.begin(), Operands.end(), O);
02779     S = new (SCEVAllocator) SCEVAddRecExpr(ID.Intern(SCEVAllocator),
02780                                            O, Operands.size(), L);
02781     UniqueSCEVs.InsertNode(S, IP);
02782   }
02783   S->setNoWrapFlags(Flags);
02784   return S;
02785 }
02786 
02787 const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS,
02788                                          const SCEV *RHS) {
02789   SmallVector<const SCEV *, 2> Ops;
02790   Ops.push_back(LHS);
02791   Ops.push_back(RHS);
02792   return getSMaxExpr(Ops);
02793 }
02794 
02795 const SCEV *
02796 ScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
02797   assert(!Ops.empty() && "Cannot get empty smax!");
02798   if (Ops.size() == 1) return Ops[0];
02799 #ifndef NDEBUG
02800   Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
02801   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
02802     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
02803            "SCEVSMaxExpr operand types don't match!");
02804 #endif
02805 
02806   // Sort by complexity, this groups all similar expression types together.
02807   GroupByComplexity(Ops, LI);
02808 
02809   // If there are any constants, fold them together.
02810   unsigned Idx = 0;
02811   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
02812     ++Idx;
02813     assert(Idx < Ops.size());
02814     while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
02815       // We found two constants, fold them together!
02816       ConstantInt *Fold = ConstantInt::get(getContext(),
02817                               APIntOps::smax(LHSC->getValue()->getValue(),
02818                                              RHSC->getValue()->getValue()));
02819       Ops[0] = getConstant(Fold);
02820       Ops.erase(Ops.begin()+1);  // Erase the folded element
02821       if (Ops.size() == 1) return Ops[0];
02822       LHSC = cast<SCEVConstant>(Ops[0]);
02823     }
02824 
02825     // If we are left with a constant minimum-int, strip it off.
02826     if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(true)) {
02827       Ops.erase(Ops.begin());
02828       --Idx;
02829     } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(true)) {
02830       // If we have an smax with a constant maximum-int, it will always be
02831       // maximum-int.
02832       return Ops[0];
02833     }
02834 
02835     if (Ops.size() == 1) return Ops[0];
02836   }
02837 
02838   // Find the first SMax
02839   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scSMaxExpr)
02840     ++Idx;
02841 
02842   // Check to see if one of the operands is an SMax. If so, expand its operands
02843   // onto our operand list, and recurse to simplify.
02844   if (Idx < Ops.size()) {
02845     bool DeletedSMax = false;
02846     while (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(Ops[Idx])) {
02847       Ops.erase(Ops.begin()+Idx);
02848       Ops.append(SMax->op_begin(), SMax->op_end());
02849       DeletedSMax = true;
02850     }
02851 
02852     if (DeletedSMax)
02853       return getSMaxExpr(Ops);
02854   }
02855 
02856   // Okay, check to see if the same value occurs in the operand list twice.  If
02857   // so, delete one.  Since we sorted the list, these values are required to
02858   // be adjacent.
02859   for (unsigned i = 0, e = Ops.size()-1; i != e; ++i)
02860     //  X smax Y smax Y  -->  X smax Y
02861     //  X smax Y         -->  X, if X is always greater than Y
02862     if (Ops[i] == Ops[i+1] ||
02863         isKnownPredicate(ICmpInst::ICMP_SGE, Ops[i], Ops[i+1])) {
02864       Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2);
02865       --i; --e;
02866     } else if (isKnownPredicate(ICmpInst::ICMP_SLE, Ops[i], Ops[i+1])) {
02867       Ops.erase(Ops.begin()+i, Ops.begin()+i+1);
02868       --i; --e;
02869     }
02870 
02871   if (Ops.size() == 1) return Ops[0];
02872 
02873   assert(!Ops.empty() && "Reduced smax down to nothing!");
02874 
02875   // Okay, it looks like we really DO need an smax expr.  Check to see if we
02876   // already have one, otherwise create a new one.
02877   FoldingSetNodeID ID;
02878   ID.AddInteger(scSMaxExpr);
02879   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02880     ID.AddPointer(Ops[i]);
02881   void *IP = nullptr;
02882   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
02883   const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
02884   std::uninitialized_copy(Ops.begin(), Ops.end(), O);
02885   SCEV *S = new (SCEVAllocator) SCEVSMaxExpr(ID.Intern(SCEVAllocator),
02886                                              O, Ops.size());
02887   UniqueSCEVs.InsertNode(S, IP);
02888   return S;
02889 }
02890 
02891 const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS,
02892                                          const SCEV *RHS) {
02893   SmallVector<const SCEV *, 2> Ops;
02894   Ops.push_back(LHS);
02895   Ops.push_back(RHS);
02896   return getUMaxExpr(Ops);
02897 }
02898 
02899 const SCEV *
02900 ScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
02901   assert(!Ops.empty() && "Cannot get empty umax!");
02902   if (Ops.size() == 1) return Ops[0];
02903 #ifndef NDEBUG
02904   Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
02905   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
02906     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
02907            "SCEVUMaxExpr operand types don't match!");
02908 #endif
02909 
02910   // Sort by complexity, this groups all similar expression types together.
02911   GroupByComplexity(Ops, LI);
02912 
02913   // If there are any constants, fold them together.
02914   unsigned Idx = 0;
02915   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
02916     ++Idx;
02917     assert(Idx < Ops.size());
02918     while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
02919       // We found two constants, fold them together!
02920       ConstantInt *Fold = ConstantInt::get(getContext(),
02921                               APIntOps::umax(LHSC->getValue()->getValue(),
02922                                              RHSC->getValue()->getValue()));
02923       Ops[0] = getConstant(Fold);
02924       Ops.erase(Ops.begin()+1);  // Erase the folded element
02925       if (Ops.size() == 1) return Ops[0];
02926       LHSC = cast<SCEVConstant>(Ops[0]);
02927     }
02928 
02929     // If we are left with a constant minimum-int, strip it off.
02930     if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(false)) {
02931       Ops.erase(Ops.begin());
02932       --Idx;
02933     } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(false)) {
02934       // If we have an umax with a constant maximum-int, it will always be
02935       // maximum-int.
02936       return Ops[0];
02937     }
02938 
02939     if (Ops.size() == 1) return Ops[0];
02940   }
02941 
02942   // Find the first UMax
02943   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scUMaxExpr)
02944     ++Idx;
02945 
02946   // Check to see if one of the operands is a UMax. If so, expand its operands
02947   // onto our operand list, and recurse to simplify.
02948   if (Idx < Ops.size()) {
02949     bool DeletedUMax = false;
02950     while (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(Ops[Idx])) {
02951       Ops.erase(Ops.begin()+Idx);
02952       Ops.append(UMax->op_begin(), UMax->op_end());
02953       DeletedUMax = true;
02954     }
02955 
02956     if (DeletedUMax)
02957       return getUMaxExpr(Ops);
02958   }
02959 
02960   // Okay, check to see if the same value occurs in the operand list twice.  If
02961   // so, delete one.  Since we sorted the list, these values are required to
02962   // be adjacent.
02963   for (unsigned i = 0, e = Ops.size()-1; i != e; ++i)
02964     //  X umax Y umax Y  -->  X umax Y
02965     //  X umax Y         -->  X, if X is always greater than Y
02966     if (Ops[i] == Ops[i+1] ||
02967         isKnownPredicate(ICmpInst::ICMP_UGE, Ops[i], Ops[i+1])) {
02968       Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2);
02969       --i; --e;
02970     } else if (isKnownPredicate(ICmpInst::ICMP_ULE, Ops[i], Ops[i+1])) {
02971       Ops.erase(Ops.begin()+i, Ops.begin()+i+1);
02972       --i; --e;
02973     }
02974 
02975   if (Ops.size() == 1) return Ops[0];
02976 
02977   assert(!Ops.empty() && "Reduced umax down to nothing!");
02978 
02979   // Okay, it looks like we really DO need a umax expr.  Check to see if we
02980   // already have one, otherwise create a new one.
02981   FoldingSetNodeID ID;
02982   ID.AddInteger(scUMaxExpr);
02983   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02984     ID.AddPointer(Ops[i]);
02985   void *IP = nullptr;
02986   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
02987   const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
02988   std::uninitialized_copy(Ops.begin(), Ops.end(), O);
02989   SCEV *S = new (SCEVAllocator) SCEVUMaxExpr(ID.Intern(SCEVAllocator),
02990                                              O, Ops.size());
02991   UniqueSCEVs.InsertNode(S, IP);
02992   return S;
02993 }
02994 
02995 const SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS,
02996                                          const SCEV *RHS) {
02997   // ~smax(~x, ~y) == smin(x, y).
02998   return getNotSCEV(getSMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS)));
02999 }
03000 
03001 const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS,
03002                                          const SCEV *RHS) {
03003   // ~umax(~x, ~y) == umin(x, y)
03004   return getNotSCEV(getUMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS)));
03005 }
03006 
03007 const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) {
03008   // If we have DataLayout, we can bypass creating a target-independent
03009   // constant expression and then folding it back into a ConstantInt.
03010   // This is just a compile-time optimization.
03011   if (DL)
03012     return getConstant(IntTy, DL->getTypeAllocSize(AllocTy));
03013 
03014   Constant *C = ConstantExpr::getSizeOf(AllocTy);
03015   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
03016     if (Constant *Folded = ConstantFoldConstantExpression(CE, DL, TLI))
03017       C = Folded;
03018   Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy));
03019   assert(Ty == IntTy && "Effective SCEV type doesn't match");
03020   return getTruncateOrZeroExtend(getSCEV(C), Ty);
03021 }
03022 
03023 const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy,
03024                                              StructType *STy,
03025                                              unsigned FieldNo) {
03026   // If we have DataLayout, we can bypass creating a target-independent
03027   // constant expression and then folding it back into a ConstantInt.
03028   // This is just a compile-time optimization.
03029   if (DL) {
03030     return getConstant(IntTy,
03031                        DL->getStructLayout(STy)->getElementOffset(FieldNo));
03032   }
03033 
03034   Constant *C = ConstantExpr::getOffsetOf(STy, FieldNo);
03035   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
03036     if (Constant *Folded = ConstantFoldConstantExpression(CE, DL, TLI))
03037       C = Folded;
03038 
03039   Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(STy));
03040   return getTruncateOrZeroExtend(getSCEV(C), Ty);
03041 }
03042 
03043 const SCEV *ScalarEvolution::getUnknown(Value *V) {
03044   // Don't attempt to do anything other than create a SCEVUnknown object
03045   // here.  createSCEV only calls getUnknown after checking for all other
03046   // interesting possibilities, and any other code that calls getUnknown
03047   // is doing so in order to hide a value from SCEV canonicalization.
03048 
03049   FoldingSetNodeID ID;
03050   ID.AddInteger(scUnknown);
03051   ID.AddPointer(V);
03052   void *IP = nullptr;
03053   if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) {
03054     assert(cast<SCEVUnknown>(S)->getValue() == V &&
03055            "Stale SCEVUnknown in uniquing map!");
03056     return S;
03057   }
03058   SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this,
03059                                             FirstUnknown);
03060   FirstUnknown = cast<SCEVUnknown>(S);
03061   UniqueSCEVs.InsertNode(S, IP);
03062   return S;
03063 }
03064 
03065 //===----------------------------------------------------------------------===//
03066 //            Basic SCEV Analysis and PHI Idiom Recognition Code
03067 //
03068 
03069 /// isSCEVable - Test if values of the given type are analyzable within
03070 /// the SCEV framework. This primarily includes integer types, and it
03071 /// can optionally include pointer types if the ScalarEvolution class
03072 /// has access to target-specific information.
03073 bool ScalarEvolution::isSCEVable(Type *Ty) const {
03074   // Integers and pointers are always SCEVable.
03075   return Ty->isIntegerTy() || Ty->isPointerTy();
03076 }
03077 
03078 /// getTypeSizeInBits - Return the size in bits of the specified type,
03079 /// for which isSCEVable must return true.
03080 uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
03081   assert(isSCEVable(Ty) && "Type is not SCEVable!");
03082 
03083   // If we have a DataLayout, use it!
03084   if (DL)
03085     return DL->getTypeSizeInBits(Ty);
03086 
03087   // Integer types have fixed sizes.
03088   if (Ty->isIntegerTy())
03089     return Ty->getPrimitiveSizeInBits();
03090 
03091   // The only other support type is pointer. Without DataLayout, conservatively
03092   // assume pointers are 64-bit.
03093   assert(Ty->isPointerTy() && "isSCEVable permitted a non-SCEVable type!");
03094   return 64;
03095 }
03096 
03097 /// getEffectiveSCEVType - Return a type with the same bitwidth as
03098 /// the given type and which represents how SCEV will treat the given
03099 /// type, for which isSCEVable must return true. For pointer types,
03100 /// this is the pointer-sized integer type.
03101 Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
03102   assert(isSCEVable(Ty) && "Type is not SCEVable!");
03103 
03104   if (Ty->isIntegerTy()) {
03105     return Ty;
03106   }
03107 
03108   // The only other support type is pointer.
03109   assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
03110 
03111   if (DL)
03112     return DL->getIntPtrType(Ty);
03113 
03114   // Without DataLayout, conservatively assume pointers are 64-bit.
03115   return Type::getInt64Ty(getContext());
03116 }
03117 
03118 const SCEV *ScalarEvolution::getCouldNotCompute() {
03119   return &CouldNotCompute;
03120 }
03121 
03122 namespace {
03123   // Helper class working with SCEVTraversal to figure out if a SCEV contains
03124   // a SCEVUnknown with null value-pointer. FindInvalidSCEVUnknown::FindOne
03125   // is set iff if find such SCEVUnknown.
03126   //
03127   struct FindInvalidSCEVUnknown {
03128     bool FindOne;
03129     FindInvalidSCEVUnknown() { FindOne = false; }
03130     bool follow(const SCEV *S) {
03131       switch (static_cast<SCEVTypes>(S->getSCEVType())) {
03132       case scConstant:
03133         return false;
03134       case scUnknown:
03135         if (!cast<SCEVUnknown>(S)->getValue())
03136           FindOne = true;
03137         return false;
03138       default:
03139         return true;
03140       }
03141     }
03142     bool isDone() const { return FindOne; }
03143   };
03144 }
03145 
03146 bool ScalarEvolution::checkValidity(const SCEV *S) const {
03147   FindInvalidSCEVUnknown F;
03148   SCEVTraversal<FindInvalidSCEVUnknown> ST(F);
03149   ST.visitAll(S);
03150 
03151   return !F.FindOne;
03152 }
03153 
03154 /// getSCEV - Return an existing SCEV if it exists, otherwise analyze the
03155 /// expression and create a new one.
03156 const SCEV *ScalarEvolution::getSCEV(Value *V) {
03157   assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
03158 
03159   ValueExprMapType::iterator I = ValueExprMap.find_as(V);
03160   if (I != ValueExprMap.end()) {
03161     const SCEV *S = I->second;
03162     if (checkValidity(S))
03163       return S;
03164     else
03165       ValueExprMap.erase(I);
03166   }
03167   const SCEV *S = createSCEV(V);
03168 
03169   // The process of creating a SCEV for V may have caused other SCEVs
03170   // to have been created, so it's necessary to insert the new entry
03171   // from scratch, rather than trying to remember the insert position
03172   // above.
03173   ValueExprMap.insert(std::make_pair(SCEVCallbackVH(V, this), S));
03174   return S;
03175 }
03176 
03177 /// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V
03178 ///
03179 const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V) {
03180   if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
03181     return getConstant(
03182                cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue())));
03183 
03184   Type *Ty = V->getType();
03185   Ty = getEffectiveSCEVType(Ty);
03186   return getMulExpr(V,
03187                   getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty))));
03188 }
03189 
03190 /// getNotSCEV - Return a SCEV corresponding to ~V = -1-V
03191 const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
03192   if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
03193     return getConstant(
03194                 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue())));
03195 
03196   Type *Ty = V->getType();
03197   Ty = getEffectiveSCEVType(Ty);
03198   const SCEV *AllOnes =
03199                    getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty)));
03200   return getMinusSCEV(AllOnes, V);
03201 }
03202 
03203 /// getMinusSCEV - Return LHS-RHS.  Minus is represented in SCEV as A+B*-1.
03204 const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS,
03205                                           SCEV::NoWrapFlags Flags) {
03206   assert(!maskFlags(Flags, SCEV::FlagNUW) && "subtraction does not have NUW");
03207 
03208   // Fast path: X - X --> 0.
03209   if (LHS == RHS)
03210     return getConstant(LHS->getType(), 0);
03211 
03212   // X - Y --> X + -Y
03213   return getAddExpr(LHS, getNegativeSCEV(RHS), Flags);
03214 }
03215 
03216 /// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the
03217 /// input value to the specified type.  If the type must be extended, it is zero
03218 /// extended.
03219 const SCEV *
03220 ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty) {
03221   Type *SrcTy = V->getType();
03222   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
03223          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
03224          "Cannot truncate or zero extend with non-integer arguments!");
03225   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
03226     return V;  // No conversion
03227   if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
03228     return getTruncateExpr(V, Ty);
03229   return getZeroExtendExpr(V, Ty);
03230 }
03231 
03232 /// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion of the
03233 /// input value to the specified type.  If the type must be extended, it is sign
03234 /// extended.
03235 const SCEV *
03236 ScalarEvolution::getTruncateOrSignExtend(const SCEV *V,
03237                                          Type *Ty) {
03238   Type *SrcTy = V->getType();
03239   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
03240          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
03241          "Cannot truncate or zero extend with non-integer arguments!");
03242   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
03243     return V;  // No conversion
03244   if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
03245     return getTruncateExpr(V, Ty);
03246   return getSignExtendExpr(V, Ty);
03247 }
03248 
03249 /// getNoopOrZeroExtend - Return a SCEV corresponding to a conversion of the
03250 /// input value to the specified type.  If the type must be extended, it is zero
03251 /// extended.  The conversion must not be narrowing.
03252 const SCEV *
03253 ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
03254   Type *SrcTy = V->getType();
03255   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
03256          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
03257          "Cannot noop or zero extend with non-integer arguments!");
03258   assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
03259          "getNoopOrZeroExtend cannot truncate!");
03260   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
03261     return V;  // No conversion
03262   return getZeroExtendExpr(V, Ty);
03263 }
03264 
03265 /// getNoopOrSignExtend - Return a SCEV corresponding to a conversion of the
03266 /// input value to the specified type.  If the type must be extended, it is sign
03267 /// extended.  The conversion must not be narrowing.
03268 const SCEV *
03269 ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
03270   Type *SrcTy = V->getType();
03271   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
03272          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
03273          "Cannot noop or sign extend with non-integer arguments!");
03274   assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
03275          "getNoopOrSignExtend cannot truncate!");
03276   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
03277     return V;  // No conversion
03278   return getSignExtendExpr(V, Ty);
03279 }
03280 
03281 /// getNoopOrAnyExtend - Return a SCEV corresponding to a conversion of
03282 /// the input value to the specified type. If the type must be extended,
03283 /// it is extended with unspecified bits. The conversion must not be
03284 /// narrowing.
03285 const SCEV *
03286 ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
03287   Type *SrcTy = V->getType();
03288   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
03289          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
03290          "Cannot noop or any extend with non-integer arguments!");
03291   assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
03292          "getNoopOrAnyExtend cannot truncate!");
03293   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
03294     return V;  // No conversion
03295   return getAnyExtendExpr(V, Ty);
03296 }
03297 
03298 /// getTruncateOrNoop - Return a SCEV corresponding to a conversion of the
03299 /// input value to the specified type.  The conversion must not be widening.
03300 const SCEV *
03301 ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
03302   Type *SrcTy = V->getType();
03303   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
03304          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
03305          "Cannot truncate or noop with non-integer arguments!");
03306   assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&
03307          "getTruncateOrNoop cannot extend!");
03308   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
03309     return V;  // No conversion
03310   return getTruncateExpr(V, Ty);
03311 }
03312 
03313 /// getUMaxFromMismatchedTypes - Promote the operands to the wider of
03314 /// the types using zero-extension, and then perform a umax operation
03315 /// with them.
03316 const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
03317                                                         const SCEV *RHS) {
03318   const SCEV *PromotedLHS = LHS;
03319   const SCEV *PromotedRHS = RHS;
03320 
03321   if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType()))
03322     PromotedRHS = getZeroExtendExpr(RHS, LHS->getType());
03323   else
03324     PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType());
03325 
03326   return getUMaxExpr(PromotedLHS, PromotedRHS);
03327 }
03328 
03329 /// getUMinFromMismatchedTypes - Promote the operands to the wider of
03330 /// the types using zero-extension, and then perform a umin operation
03331 /// with them.
03332 const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
03333                                                         const SCEV *RHS) {
03334   const SCEV *PromotedLHS = LHS;
03335   const SCEV *PromotedRHS = RHS;
03336 
03337   if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType()))
03338     PromotedRHS = getZeroExtendExpr(RHS, LHS->getType());
03339   else
03340     PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType());
03341 
03342   return getUMinExpr(PromotedLHS, PromotedRHS);
03343 }
03344 
03345 /// getPointerBase - Transitively follow the chain of pointer-type operands
03346 /// until reaching a SCEV that does not have a single pointer operand. This
03347 /// returns a SCEVUnknown pointer for well-formed pointer-type expressions,
03348 /// but corner cases do exist.
03349 const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
03350   // A pointer operand may evaluate to a nonpointer expression, such as null.
03351   if (!V->getType()->isPointerTy())
03352     return V;
03353 
03354   if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(V)) {
03355     return getPointerBase(Cast->getOperand());
03356   }
03357   else if (const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(V)) {
03358     const SCEV *PtrOp = nullptr;
03359     for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end();
03360          I != E; ++I) {
03361       if ((*I)->getType()->isPointerTy()) {
03362         // Cannot find the base of an expression with multiple pointer operands.
03363         if (PtrOp)
03364           return V;
03365         PtrOp = *I;
03366       }
03367     }
03368     if (!PtrOp)
03369       return V;
03370     return getPointerBase(PtrOp);
03371   }
03372   return V;
03373 }
03374 
03375 /// PushDefUseChildren - Push users of the given Instruction
03376 /// onto the given Worklist.
03377 static void
03378 PushDefUseChildren(Instruction *I,
03379                    SmallVectorImpl<Instruction *> &Worklist) {
03380   // Push the def-use children onto the Worklist stack.
03381   for (User *U : I->users())
03382     Worklist.push_back(cast<Instruction>(U));
03383 }
03384 
03385 /// ForgetSymbolicValue - This looks up computed SCEV values for all
03386 /// instructions that depend on the given instruction and removes them from
03387 /// the ValueExprMapType map if they reference SymName. This is used during PHI
03388 /// resolution.
03389 void
03390 ScalarEvolution::ForgetSymbolicName(Instruction *PN, const SCEV *SymName) {
03391   SmallVector<Instruction *, 16> Worklist;
03392   PushDefUseChildren(PN, Worklist);
03393 
03394   SmallPtrSet<Instruction *, 8> Visited;
03395   Visited.insert(PN);
03396   while (!Worklist.empty()) {
03397     Instruction *I = Worklist.pop_back_val();
03398     if (!Visited.insert(I).second)
03399       continue;
03400 
03401     ValueExprMapType::iterator It =
03402       ValueExprMap.find_as(static_cast<Value *>(I));
03403     if (It != ValueExprMap.end()) {
03404       const SCEV *Old = It->second;
03405 
03406       // Short-circuit the def-use traversal if the symbolic name
03407       // ceases to appear in expressions.
03408       if (Old != SymName && !hasOperand(Old, SymName))
03409         continue;
03410 
03411       // SCEVUnknown for a PHI either means that it has an unrecognized
03412       // structure, it's a PHI that's in the progress of being computed
03413       // by createNodeForPHI, or it's a single-value PHI. In the first case,
03414       // additional loop trip count information isn't going to change anything.
03415       // In the second case, createNodeForPHI will perform the necessary
03416       // updates on its own when it gets to that point. In the third, we do
03417       // want to forget the SCEVUnknown.
03418       if (!isa<PHINode>(I) ||
03419           !isa<SCEVUnknown>(Old) ||
03420           (I != PN && Old == SymName)) {
03421         forgetMemoizedResults(Old);
03422         ValueExprMap.erase(It);
03423       }
03424     }
03425 
03426     PushDefUseChildren(I, Worklist);
03427   }
03428 }
03429 
03430 /// createNodeForPHI - PHI nodes have two cases.  Either the PHI node exists in
03431 /// a loop header, making it a potential recurrence, or it doesn't.
03432 ///
03433 const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
03434   if (const Loop *L = LI->getLoopFor(PN->getParent()))
03435     if (L->getHeader() == PN->getParent()) {
03436       // The loop may have multiple entrances or multiple exits; we can analyze
03437       // this phi as an addrec if it has a unique entry value and a unique
03438       // backedge value.
03439       Value *BEValueV = nullptr, *StartValueV = nullptr;
03440       for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
03441         Value *V = PN->getIncomingValue(i);
03442         if (L->contains(PN->getIncomingBlock(i))) {
03443           if (!BEValueV) {
03444             BEValueV = V;
03445           } else if (BEValueV != V) {
03446             BEValueV = nullptr;
03447             break;
03448           }
03449         } else if (!StartValueV) {
03450           StartValueV = V;
03451         } else if (StartValueV != V) {
03452           StartValueV = nullptr;
03453           break;
03454         }
03455       }
03456       if (BEValueV && StartValueV) {
03457         // While we are analyzing this PHI node, handle its value symbolically.
03458         const SCEV *SymbolicName = getUnknown(PN);
03459         assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
03460                "PHI node already processed?");
03461         ValueExprMap.insert(std::make_pair(SCEVCallbackVH(PN, this), SymbolicName));
03462 
03463         // Using this symbolic name for the PHI, analyze the value coming around
03464         // the back-edge.
03465         const SCEV *BEValue = getSCEV(BEValueV);
03466 
03467         // NOTE: If BEValue is loop invariant, we know that the PHI node just
03468         // has a special value for the first iteration of the loop.
03469 
03470         // If the value coming around the backedge is an add with the symbolic
03471         // value we just inserted, then we found a simple induction variable!
03472         if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) {
03473           // If there is a single occurrence of the symbolic value, replace it
03474           // with a recurrence.
03475           unsigned FoundIndex = Add->getNumOperands();
03476           for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
03477             if (Add->getOperand(i) == SymbolicName)
03478               if (FoundIndex == e) {
03479                 FoundIndex = i;
03480                 break;
03481               }
03482 
03483           if (FoundIndex != Add->getNumOperands()) {
03484             // Create an add with everything but the specified operand.
03485             SmallVector<const SCEV *, 8> Ops;
03486             for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
03487               if (i != FoundIndex)
03488                 Ops.push_back(Add->getOperand(i));
03489             const SCEV *Accum = getAddExpr(Ops);
03490 
03491             // This is not a valid addrec if the step amount is varying each
03492             // loop iteration, but is not itself an addrec in this loop.
03493             if (isLoopInvariant(Accum, L) ||
03494                 (isa<SCEVAddRecExpr>(Accum) &&
03495                  cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) {
03496               SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
03497 
03498               // If the increment doesn't overflow, then neither the addrec nor
03499               // the post-increment will overflow.
03500               if (const AddOperator *OBO = dyn_cast<AddOperator>(BEValueV)) {
03501                 if (OBO->hasNoUnsignedWrap())
03502                   Flags = setFlags(Flags, SCEV::FlagNUW);
03503                 if (OBO->hasNoSignedWrap())
03504                   Flags = setFlags(Flags, SCEV::FlagNSW);
03505               } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(BEValueV)) {
03506                 // If the increment is an inbounds GEP, then we know the address
03507                 // space cannot be wrapped around. We cannot make any guarantee
03508                 // about signed or unsigned overflow because pointers are
03509                 // unsigned but we may have a negative index from the base
03510                 // pointer. We can guarantee that no unsigned wrap occurs if the
03511                 // indices form a positive value.
03512                 if (GEP->isInBounds()) {
03513                   Flags = setFlags(Flags, SCEV::FlagNW);
03514 
03515                   const SCEV *Ptr = getSCEV(GEP->getPointerOperand());
03516                   if (isKnownPositive(getMinusSCEV(getSCEV(GEP), Ptr)))
03517                     Flags = setFlags(Flags, SCEV::FlagNUW);
03518                 }
03519               } else if (const SubOperator *OBO =
03520                            dyn_cast<SubOperator>(BEValueV)) {
03521                 if (OBO->hasNoUnsignedWrap())
03522                   Flags = setFlags(Flags, SCEV::FlagNUW);
03523                 if (OBO->hasNoSignedWrap())
03524                   Flags = setFlags(Flags, SCEV::FlagNSW);
03525               }
03526 
03527               const SCEV *StartVal = getSCEV(StartValueV);
03528               const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags);
03529 
03530               // Since the no-wrap flags are on the increment, they apply to the
03531               // post-incremented value as well.
03532               if (isLoopInvariant(Accum, L))
03533                 (void)getAddRecExpr(getAddExpr(StartVal, Accum),
03534                                     Accum, L, Flags);
03535 
03536               // Okay, for the entire analysis of this edge we assumed the PHI
03537               // to be symbolic.  We now need to go back and purge all of the
03538               // entries for the scalars that use the symbolic expression.
03539               ForgetSymbolicName(PN, SymbolicName);
03540               ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV;
03541               return PHISCEV;
03542             }
03543           }
03544         } else if (const SCEVAddRecExpr *AddRec =
03545                      dyn_cast<SCEVAddRecExpr>(BEValue)) {
03546           // Otherwise, this could be a loop like this:
03547           //     i = 0;  for (j = 1; ..; ++j) { ....  i = j; }
03548           // In this case, j = {1,+,1}  and BEValue is j.
03549           // Because the other in-value of i (0) fits the evolution of BEValue
03550           // i really is an addrec evolution.
03551           if (AddRec->getLoop() == L && AddRec->isAffine()) {
03552             const SCEV *StartVal = getSCEV(StartValueV);
03553 
03554             // If StartVal = j.start - j.stride, we can use StartVal as the
03555             // initial step of the addrec evolution.
03556             if (StartVal == getMinusSCEV(AddRec->getOperand(0),
03557                                          AddRec->getOperand(1))) {
03558               // FIXME: For constant StartVal, we should be able to infer
03559               // no-wrap flags.
03560               const SCEV *PHISCEV =
03561                 getAddRecExpr(StartVal, AddRec->getOperand(1), L,
03562                               SCEV::FlagAnyWrap);
03563 
03564               // Okay, for the entire analysis of this edge we assumed the PHI
03565               // to be symbolic.  We now need to go back and purge all of the
03566               // entries for the scalars that use the symbolic expression.
03567               ForgetSymbolicName(PN, SymbolicName);
03568               ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV;
03569               return PHISCEV;
03570             }
03571           }
03572         }
03573       }
03574     }
03575 
03576   // If the PHI has a single incoming value, follow that value, unless the
03577   // PHI's incoming blocks are in a different loop, in which case doing so
03578   // risks breaking LCSSA form. Instcombine would normally zap these, but
03579   // it doesn't have DominatorTree information, so it may miss cases.
03580   if (Value *V = SimplifyInstruction(PN, DL, TLI, DT, AT))
03581     if (LI->replacementPreservesLCSSAForm(PN, V))
03582       return getSCEV(V);
03583 
03584   // If it's not a loop phi, we can't handle it yet.
03585   return getUnknown(PN);
03586 }
03587 
03588 /// createNodeForGEP - Expand GEP instructions into add and multiply
03589 /// operations. This allows them to be analyzed by regular SCEV code.
03590 ///
03591 const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
03592   Type *IntPtrTy = getEffectiveSCEVType(GEP->getType());
03593   Value *Base = GEP->getOperand(0);
03594   // Don't attempt to analyze GEPs over unsized objects.
03595   if (!Base->getType()->getPointerElementType()->isSized())
03596     return getUnknown(GEP);
03597 
03598   // Don't blindly transfer the inbounds flag from the GEP instruction to the
03599   // Add expression, because the Instruction may be guarded by control flow
03600   // and the no-overflow bits may not be valid for the expression in any
03601   // context.
03602   SCEV::NoWrapFlags Wrap = GEP->isInBounds() ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
03603 
03604   const SCEV *TotalOffset = getConstant(IntPtrTy, 0);
03605   gep_type_iterator GTI = gep_type_begin(GEP);
03606   for (GetElementPtrInst::op_iterator I = std::next(GEP->op_begin()),
03607                                       E = GEP->op_end();
03608        I != E; ++I) {
03609     Value *Index = *I;
03610     // Compute the (potentially symbolic) offset in bytes for this index.
03611     if (StructType *STy = dyn_cast<StructType>(*GTI++)) {
03612       // For a struct, add the member offset.
03613       unsigned FieldNo = cast<ConstantInt>(Index)->getZExtValue();
03614       const SCEV *FieldOffset = getOffsetOfExpr(IntPtrTy, STy, FieldNo);
03615 
03616       // Add the field offset to the running total offset.
03617       TotalOffset = getAddExpr(TotalOffset, FieldOffset);
03618     } else {
03619       // For an array, add the element offset, explicitly scaled.
03620       const SCEV *ElementSize = getSizeOfExpr(IntPtrTy, *GTI);
03621       const SCEV *IndexS = getSCEV(Index);
03622       // Getelementptr indices are signed.
03623       IndexS = getTruncateOrSignExtend(IndexS, IntPtrTy);
03624 
03625       // Multiply the index by the element size to compute the element offset.
03626       const SCEV *LocalOffset = getMulExpr(IndexS, ElementSize, Wrap);
03627 
03628       // Add the element offset to the running total offset.
03629       TotalOffset = getAddExpr(TotalOffset, LocalOffset);
03630     }
03631   }
03632 
03633   // Get the SCEV for the GEP base.
03634   const SCEV *BaseS = getSCEV(Base);
03635 
03636   // Add the total offset from all the GEP indices to the base.
03637   return getAddExpr(BaseS, TotalOffset, Wrap);
03638 }
03639 
03640 /// GetMinTrailingZeros - Determine the minimum number of zero bits that S is
03641 /// guaranteed to end in (at every loop iteration).  It is, at the same time,
03642 /// the minimum number of times S is divisible by 2.  For example, given {4,+,8}
03643 /// it returns 2.  If S is guaranteed to be 0, it returns the bitwidth of S.
03644 uint32_t
03645 ScalarEvolution::GetMinTrailingZeros(const SCEV *S) {
03646   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
03647     return C->getValue()->getValue().countTrailingZeros();
03648 
03649   if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(S))
03650     return std::min(GetMinTrailingZeros(T->getOperand()),
03651                     (uint32_t)getTypeSizeInBits(T->getType()));
03652 
03653   if (const SCEVZeroExtendExpr *E = dyn_cast<SCEVZeroExtendExpr>(S)) {
03654     uint32_t OpRes = GetMinTrailingZeros(E->getOperand());
03655     return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ?
03656              getTypeSizeInBits(E->getType()) : OpRes;
03657   }
03658 
03659   if (const SCEVSignExtendExpr *E = dyn_cast<SCEVSignExtendExpr>(S)) {
03660     uint32_t OpRes = GetMinTrailingZeros(E->getOperand());
03661     return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ?
03662              getTypeSizeInBits(E->getType()) : OpRes;
03663   }
03664 
03665   if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(S)) {
03666     // The result is the min of all operands results.
03667     uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0));
03668     for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i)
03669       MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i)));
03670     return MinOpRes;
03671   }
03672 
03673   if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) {
03674     // The result is the sum of all operands results.
03675     uint32_t SumOpRes = GetMinTrailingZeros(M->getOperand(0));
03676     uint32_t BitWidth = getTypeSizeInBits(M->getType());
03677     for (unsigned i = 1, e = M->getNumOperands();
03678          SumOpRes != BitWidth && i != e; ++i)
03679       SumOpRes = std::min(SumOpRes + GetMinTrailingZeros(M->getOperand(i)),
03680                           BitWidth);
03681     return SumOpRes;
03682   }
03683 
03684   if (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(S)) {
03685     // The result is the min of all operands results.
03686     uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0));
03687     for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i)
03688       MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i)));
03689     return MinOpRes;
03690   }
03691 
03692   if (const SCEVSMaxExpr *M = dyn_cast<SCEVSMaxExpr>(S)) {
03693     // The result is the min of all operands results.
03694     uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0));
03695     for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i)
03696       MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i)));
03697     return MinOpRes;
03698   }
03699 
03700   if (const SCEVUMaxExpr *M = dyn_cast<SCEVUMaxExpr>(S)) {
03701     // The result is the min of all operands results.
03702     uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0));
03703     for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i)
03704       MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i)));
03705     return MinOpRes;
03706   }
03707 
03708   if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
03709     // For a SCEVUnknown, ask ValueTracking.
03710     unsigned BitWidth = getTypeSizeInBits(U->getType());
03711     APInt Zeros(BitWidth, 0), Ones(BitWidth, 0);
03712     computeKnownBits(U->getValue(), Zeros, Ones, DL, 0, AT, nullptr, DT);
03713     return Zeros.countTrailingOnes();
03714   }
03715 
03716   // SCEVUDivExpr
03717   return 0;
03718 }
03719 
03720 /// GetRangeFromMetadata - Helper method to assign a range to V from
03721 /// metadata present in the IR.
03722 static Optional<ConstantRange> GetRangeFromMetadata(Value *V) {
03723   if (Instruction *I = dyn_cast<Instruction>(V)) {
03724     if (MDNode *MD = I->getMetadata(LLVMContext::MD_range)) {
03725       ConstantRange TotalRange(
03726           cast<IntegerType>(I->getType())->getBitWidth(), false);
03727 
03728       unsigned NumRanges = MD->getNumOperands() / 2;
03729       assert(NumRanges >= 1);
03730 
03731       for (unsigned i = 0; i < NumRanges; ++i) {
03732         ConstantInt *Lower = cast<ConstantInt>(MD->getOperand(2*i + 0));
03733         ConstantInt *Upper = cast<ConstantInt>(MD->getOperand(2*i + 1));
03734         ConstantRange Range(Lower->getValue(), Upper->getValue());
03735         TotalRange = TotalRange.unionWith(Range);
03736       }
03737 
03738       return TotalRange;
03739     }
03740   }
03741 
03742   return None;
03743 }
03744 
03745 /// getUnsignedRange - Determine the unsigned range for a particular SCEV.
03746 ///
03747 ConstantRange
03748 ScalarEvolution::getUnsignedRange(const SCEV *S) {
03749   // See if we've computed this range already.
03750   DenseMap<const SCEV *, ConstantRange>::iterator I = UnsignedRanges.find(S);
03751   if (I != UnsignedRanges.end())
03752     return I->second;
03753 
03754   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
03755     return setUnsignedRange(C, ConstantRange(C->getValue()->getValue()));
03756 
03757   unsigned BitWidth = getTypeSizeInBits(S->getType());
03758   ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
03759 
03760   // If the value has known zeros, the maximum unsigned value will have those
03761   // known zeros as well.
03762   uint32_t TZ = GetMinTrailingZeros(S);
03763   if (TZ != 0)
03764     ConservativeResult =
03765       ConstantRange(APInt::getMinValue(BitWidth),
03766                     APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1);
03767 
03768   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
03769     ConstantRange X = getUnsignedRange(Add->getOperand(0));
03770     for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i)
03771       X = X.add(getUnsignedRange(Add->getOperand(i)));
03772     return setUnsignedRange(Add, ConservativeResult.intersectWith(X));
03773   }
03774 
03775   if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
03776     ConstantRange X = getUnsignedRange(Mul->getOperand(0));
03777     for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i)
03778       X = X.multiply(getUnsignedRange(Mul->getOperand(i)));
03779     return setUnsignedRange(Mul, ConservativeResult.intersectWith(X));
03780   }
03781 
03782   if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) {
03783     ConstantRange X = getUnsignedRange(SMax->getOperand(0));
03784     for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i)
03785       X = X.smax(getUnsignedRange(SMax->getOperand(i)));
03786     return setUnsignedRange(SMax, ConservativeResult.intersectWith(X));
03787   }
03788 
03789   if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) {
03790     ConstantRange X = getUnsignedRange(UMax->getOperand(0));
03791     for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i)
03792       X = X.umax(getUnsignedRange(UMax->getOperand(i)));
03793     return setUnsignedRange(UMax, ConservativeResult.intersectWith(X));
03794   }
03795 
03796   if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) {
03797     ConstantRange X = getUnsignedRange(UDiv->getLHS());
03798     ConstantRange Y = getUnsignedRange(UDiv->getRHS());
03799     return setUnsignedRange(UDiv, ConservativeResult.intersectWith(X.udiv(Y)));
03800   }
03801 
03802   if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) {
03803     ConstantRange X = getUnsignedRange(ZExt->getOperand());
03804     return setUnsignedRange(ZExt,
03805       ConservativeResult.intersectWith(X.zeroExtend(BitWidth)));
03806   }
03807 
03808   if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) {
03809     ConstantRange X = getUnsignedRange(SExt->getOperand());
03810     return setUnsignedRange(SExt,
03811       ConservativeResult.intersectWith(X.signExtend(BitWidth)));
03812   }
03813 
03814   if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) {
03815     ConstantRange X = getUnsignedRange(Trunc->getOperand());
03816     return setUnsignedRange(Trunc,
03817       ConservativeResult.intersectWith(X.truncate(BitWidth)));
03818   }
03819 
03820   if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
03821     // If there's no unsigned wrap, the value will never be less than its
03822     // initial value.
03823     if (AddRec->getNoWrapFlags(SCEV::FlagNUW))
03824       if (const SCEVConstant *C = dyn_cast<SCEVConstant>(AddRec->getStart()))
03825         if (!C->getValue()->isZero())
03826           ConservativeResult =
03827             ConservativeResult.intersectWith(
03828               ConstantRange(C->getValue()->getValue(), APInt(BitWidth, 0)));
03829 
03830     // TODO: non-affine addrec
03831     if (AddRec->isAffine()) {
03832       Type *Ty = AddRec->getType();
03833       const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop());
03834       if (!isa<SCEVCouldNotCompute>(MaxBECount) &&
03835           getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) {
03836         MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty);
03837 
03838         const SCEV *Start = AddRec->getStart();
03839         const SCEV *Step = AddRec->getStepRecurrence(*this);
03840 
03841         ConstantRange StartRange = getUnsignedRange(Start);
03842         ConstantRange StepRange = getSignedRange(Step);
03843         ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount);
03844         ConstantRange EndRange =
03845           StartRange.add(MaxBECountRange.multiply(StepRange));
03846 
03847         // Check for overflow. This must be done with ConstantRange arithmetic
03848         // because we could be called from within the ScalarEvolution overflow
03849         // checking code.
03850         ConstantRange ExtStartRange = StartRange.zextOrTrunc(BitWidth*2+1);
03851         ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1);
03852         ConstantRange ExtMaxBECountRange =
03853           MaxBECountRange.zextOrTrunc(BitWidth*2+1);
03854         ConstantRange ExtEndRange = EndRange.zextOrTrunc(BitWidth*2+1);
03855         if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) !=
03856             ExtEndRange)
03857           return setUnsignedRange(AddRec, ConservativeResult);
03858 
03859         APInt Min = APIntOps::umin(StartRange.getUnsignedMin(),
03860                                    EndRange.getUnsignedMin());
03861         APInt Max = APIntOps::umax(StartRange.getUnsignedMax(),
03862                                    EndRange.getUnsignedMax());
03863         if (Min.isMinValue() && Max.isMaxValue())
03864           return setUnsignedRange(AddRec, ConservativeResult);
03865         return setUnsignedRange(AddRec,
03866           ConservativeResult.intersectWith(ConstantRange(Min, Max+1)));
03867       }
03868     }
03869 
03870     return setUnsignedRange(AddRec, ConservativeResult);
03871   }
03872 
03873   if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
03874     // Check if the IR explicitly contains !range metadata.
03875     Optional<ConstantRange> MDRange = GetRangeFromMetadata(U->getValue());
03876     if (MDRange.hasValue())
03877       ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue());
03878 
03879     // For a SCEVUnknown, ask ValueTracking.
03880     APInt Zeros(BitWidth, 0), Ones(BitWidth, 0);
03881     computeKnownBits(U->getValue(), Zeros, Ones, DL, 0, AT, nullptr, DT);
03882     if (Ones == ~Zeros + 1)
03883       return setUnsignedRange(U, ConservativeResult);
03884     return setUnsignedRange(U,
03885       ConservativeResult.intersectWith(ConstantRange(Ones, ~Zeros + 1)));
03886   }
03887 
03888   return setUnsignedRange(S, ConservativeResult);
03889 }
03890 
03891 /// getSignedRange - Determine the signed range for a particular SCEV.
03892 ///
03893 ConstantRange
03894 ScalarEvolution::getSignedRange(const SCEV *S) {
03895   // See if we've computed this range already.
03896   DenseMap<const SCEV *, ConstantRange>::iterator I = SignedRanges.find(S);
03897   if (I != SignedRanges.end())
03898     return I->second;
03899 
03900   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
03901     return setSignedRange(C, ConstantRange(C->getValue()->getValue()));
03902 
03903   unsigned BitWidth = getTypeSizeInBits(S->getType());
03904   ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
03905 
03906   // If the value has known zeros, the maximum signed value will have those
03907   // known zeros as well.
03908   uint32_t TZ = GetMinTrailingZeros(S);
03909   if (TZ != 0)
03910     ConservativeResult =
03911       ConstantRange(APInt::getSignedMinValue(BitWidth),
03912                     APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1);
03913 
03914   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
03915     ConstantRange X = getSignedRange(Add->getOperand(0));
03916     for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i)
03917       X = X.add(getSignedRange(Add->getOperand(i)));
03918     return setSignedRange(Add, ConservativeResult.intersectWith(X));
03919   }
03920 
03921   if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
03922     ConstantRange X = getSignedRange(Mul->getOperand(0));
03923     for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i)
03924       X = X.multiply(getSignedRange(Mul->getOperand(i)));
03925     return setSignedRange(Mul, ConservativeResult.intersectWith(X));
03926   }
03927 
03928   if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) {
03929     ConstantRange X = getSignedRange(SMax->getOperand(0));
03930     for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i)
03931       X = X.smax(getSignedRange(SMax->getOperand(i)));
03932     return setSignedRange(SMax, ConservativeResult.intersectWith(X));
03933   }
03934 
03935   if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) {
03936     ConstantRange X = getSignedRange(UMax->getOperand(0));
03937     for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i)
03938       X = X.umax(getSignedRange(UMax->getOperand(i)));
03939     return setSignedRange(UMax, ConservativeResult.intersectWith(X));
03940   }
03941 
03942   if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) {
03943     ConstantRange X = getSignedRange(UDiv->getLHS());
03944     ConstantRange Y = getSignedRange(UDiv->getRHS());
03945     return setSignedRange(UDiv, ConservativeResult.intersectWith(X.udiv(Y)));
03946   }
03947 
03948   if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) {
03949     ConstantRange X = getSignedRange(ZExt->getOperand());
03950     return setSignedRange(ZExt,
03951       ConservativeResult.intersectWith(X.zeroExtend(BitWidth)));
03952   }
03953 
03954   if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) {
03955     ConstantRange X = getSignedRange(SExt->getOperand());
03956     return setSignedRange(SExt,
03957       ConservativeResult.intersectWith(X.signExtend(BitWidth)));
03958   }
03959 
03960   if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) {
03961     ConstantRange X = getSignedRange(Trunc->getOperand());
03962     return setSignedRange(Trunc,
03963       ConservativeResult.intersectWith(X.truncate(BitWidth)));
03964   }
03965 
03966   if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
03967     // If there's no signed wrap, and all the operands have the same sign or
03968     // zero, the value won't ever change sign.
03969     if (AddRec->getNoWrapFlags(SCEV::FlagNSW)) {
03970       bool AllNonNeg = true;
03971       bool AllNonPos = true;
03972       for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
03973         if (!isKnownNonNegative(AddRec->getOperand(i))) AllNonNeg = false;
03974         if (!isKnownNonPositive(AddRec->getOperand(i))) AllNonPos = false;
03975       }
03976       if (AllNonNeg)
03977         ConservativeResult = ConservativeResult.intersectWith(
03978           ConstantRange(APInt(BitWidth, 0),
03979                         APInt::getSignedMinValue(BitWidth)));
03980       else if (AllNonPos)
03981         ConservativeResult = ConservativeResult.intersectWith(
03982           ConstantRange(APInt::getSignedMinValue(BitWidth),
03983                         APInt(BitWidth, 1)));
03984     }
03985 
03986     // TODO: non-affine addrec
03987     if (AddRec->isAffine()) {
03988       Type *Ty = AddRec->getType();
03989       const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop());
03990       if (!isa<SCEVCouldNotCompute>(MaxBECount) &&
03991           getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) {
03992         MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty);
03993 
03994         const SCEV *Start = AddRec->getStart();
03995         const SCEV *Step = AddRec->getStepRecurrence(*this);
03996 
03997         ConstantRange StartRange = getSignedRange(Start);
03998         ConstantRange StepRange = getSignedRange(Step);
03999         ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount);
04000         ConstantRange EndRange =
04001           StartRange.add(MaxBECountRange.multiply(StepRange));
04002 
04003         // Check for overflow. This must be done with ConstantRange arithmetic
04004         // because we could be called from within the ScalarEvolution overflow
04005         // checking code.
04006         ConstantRange ExtStartRange = StartRange.sextOrTrunc(BitWidth*2+1);
04007         ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1);
04008         ConstantRange ExtMaxBECountRange =
04009           MaxBECountRange.zextOrTrunc(BitWidth*2+1);
04010         ConstantRange ExtEndRange = EndRange.sextOrTrunc(BitWidth*2+1);
04011         if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) !=
04012             ExtEndRange)
04013           return setSignedRange(AddRec, ConservativeResult);
04014 
04015         APInt Min = APIntOps::smin(StartRange.getSignedMin(),
04016                                    EndRange.getSignedMin());
04017         APInt Max = APIntOps::smax(StartRange.getSignedMax(),
04018                                    EndRange.getSignedMax());
04019         if (Min.isMinSignedValue() && Max.isMaxSignedValue())
04020           return setSignedRange(AddRec, ConservativeResult);
04021         return setSignedRange(AddRec,
04022           ConservativeResult.intersectWith(ConstantRange(Min, Max+1)));
04023       }
04024     }
04025 
04026     return setSignedRange(AddRec, ConservativeResult);
04027   }
04028 
04029   if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
04030     // Check if the IR explicitly contains !range metadata.
04031     Optional<ConstantRange> MDRange = GetRangeFromMetadata(U->getValue());
04032     if (MDRange.hasValue())
04033       ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue());
04034 
04035     // For a SCEVUnknown, ask ValueTracking.
04036     if (!U->getValue()->getType()->isIntegerTy() && !DL)
04037       return setSignedRange(U, ConservativeResult);
04038     unsigned NS = ComputeNumSignBits(U->getValue(), DL, 0, AT, nullptr, DT);
04039     if (NS <= 1)
04040       return setSignedRange(U, ConservativeResult);
04041     return setSignedRange(U, ConservativeResult.intersectWith(
04042       ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1),
04043                     APInt::getSignedMaxValue(BitWidth).ashr(NS - 1)+1)));
04044   }
04045 
04046   return setSignedRange(S, ConservativeResult);
04047 }
04048 
04049 /// createSCEV - We know that there is no SCEV for the specified value.
04050 /// Analyze the expression.
04051 ///
04052 const SCEV *ScalarEvolution::createSCEV(Value *V) {
04053   if (!isSCEVable(V->getType()))
04054     return getUnknown(V);
04055 
04056   unsigned Opcode = Instruction::UserOp1;
04057   if (Instruction *I = dyn_cast<Instruction>(V)) {
04058     Opcode = I->getOpcode();
04059 
04060     // Don't attempt to analyze instructions in blocks that aren't
04061     // reachable. Such instructions don't matter, and they aren't required
04062     // to obey basic rules for definitions dominating uses which this
04063     // analysis depends on.
04064     if (!DT->isReachableFromEntry(I->getParent()))
04065       return getUnknown(V);
04066   } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
04067     Opcode = CE->getOpcode();
04068   else if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
04069     return getConstant(CI);
04070   else if (isa<ConstantPointerNull>(V))
04071     return getConstant(V->getType(), 0);
04072   else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V))
04073     return GA->mayBeOverridden() ? getUnknown(V) : getSCEV(GA->getAliasee());
04074   else
04075     return getUnknown(V);
04076 
04077   Operator *U = cast<Operator>(V);
04078   switch (Opcode) {
04079   case Instruction::Add: {
04080     // The simple thing to do would be to just call getSCEV on both operands
04081     // and call getAddExpr with the result. However if we're looking at a
04082     // bunch of things all added together, this can be quite inefficient,
04083     // because it leads to N-1 getAddExpr calls for N ultimate operands.
04084     // Instead, gather up all the operands and make a single getAddExpr call.
04085     // LLVM IR canonical form means we need only traverse the left operands.
04086     //
04087     // Don't apply this instruction's NSW or NUW flags to the new
04088     // expression. The instruction may be guarded by control flow that the
04089     // no-wrap behavior depends on. Non-control-equivalent instructions can be
04090     // mapped to the same SCEV expression, and it would be incorrect to transfer
04091     // NSW/NUW semantics to those operations.
04092     SmallVector<const SCEV *, 4> AddOps;
04093     AddOps.push_back(getSCEV(U->getOperand(1)));
04094     for (Value *Op = U->getOperand(0); ; Op = U->getOperand(0)) {
04095       unsigned Opcode = Op->getValueID() - Value::InstructionVal;
04096       if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
04097         break;
04098       U = cast<Operator>(Op);
04099       const SCEV *Op1 = getSCEV(U->getOperand(1));
04100       if (Opcode == Instruction::Sub)
04101         AddOps.push_back(getNegativeSCEV(Op1));
04102       else
04103         AddOps.push_back(Op1);
04104     }
04105     AddOps.push_back(getSCEV(U->getOperand(0)));
04106     return getAddExpr(AddOps);
04107   }
04108   case Instruction::Mul: {
04109     // Don't transfer NSW/NUW for the same reason as AddExpr.
04110     SmallVector<const SCEV *, 4> MulOps;
04111     MulOps.push_back(getSCEV(U->getOperand(1)));
04112     for (Value *Op = U->getOperand(0);
04113          Op->getValueID() == Instruction::Mul + Value::InstructionVal;
04114          Op = U->getOperand(0)) {
04115       U = cast<Operator>(Op);
04116       MulOps.push_back(getSCEV(U->getOperand(1)));
04117     }
04118     MulOps.push_back(getSCEV(U->getOperand(0)));
04119     return getMulExpr(MulOps);
04120   }
04121   case Instruction::UDiv:
04122     return getUDivExpr(getSCEV(U->getOperand(0)),
04123                        getSCEV(U->getOperand(1)));
04124   case Instruction::Sub:
04125     return getMinusSCEV(getSCEV(U->getOperand(0)),
04126                         getSCEV(U->getOperand(1)));
04127   case Instruction::And:
04128     // For an expression like x&255 that merely masks off the high bits,
04129     // use zext(trunc(x)) as the SCEV expression.
04130     if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
04131       if (CI->isNullValue())
04132         return getSCEV(U->getOperand(1));
04133       if (CI->isAllOnesValue())
04134         return getSCEV(U->getOperand(0));
04135       const APInt &A = CI->getValue();
04136 
04137       // Instcombine's ShrinkDemandedConstant may strip bits out of
04138       // constants, obscuring what would otherwise be a low-bits mask.
04139       // Use computeKnownBits to compute what ShrinkDemandedConstant
04140       // knew about to reconstruct a low-bits mask value.
04141       unsigned LZ = A.countLeadingZeros();
04142       unsigned TZ = A.countTrailingZeros();
04143       unsigned BitWidth = A.getBitWidth();
04144       APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0);
04145       computeKnownBits(U->getOperand(0), KnownZero, KnownOne, DL,
04146                        0, AT, nullptr, DT);
04147 
04148       APInt EffectiveMask =
04149           APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ);
04150       if ((LZ != 0 || TZ != 0) && !((~A & ~KnownZero) & EffectiveMask)) {
04151         const SCEV *MulCount = getConstant(
04152             ConstantInt::get(getContext(), APInt::getOneBitSet(BitWidth, TZ)));
04153         return getMulExpr(
04154             getZeroExtendExpr(
04155                 getTruncateExpr(
04156                     getUDivExactExpr(getSCEV(U->getOperand(0)), MulCount),
04157                     IntegerType::get(getContext(), BitWidth - LZ - TZ)),
04158                 U->getType()),
04159             MulCount);
04160       }
04161     }
04162     break;
04163 
04164   case Instruction::Or:
04165     // If the RHS of the Or is a constant, we may have something like:
04166     // X*4+1 which got turned into X*4|1.  Handle this as an Add so loop
04167     // optimizations will transparently handle this case.
04168     //
04169     // In order for this transformation to be safe, the LHS must be of the
04170     // form X*(2^n) and the Or constant must be less than 2^n.
04171     if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
04172       const SCEV *LHS = getSCEV(U->getOperand(0));
04173       const APInt &CIVal = CI->getValue();
04174       if (GetMinTrailingZeros(LHS) >=
04175           (CIVal.getBitWidth() - CIVal.countLeadingZeros())) {
04176         // Build a plain add SCEV.
04177         const SCEV *S = getAddExpr(LHS, getSCEV(CI));
04178         // If the LHS of the add was an addrec and it has no-wrap flags,
04179         // transfer the no-wrap flags, since an or won't introduce a wrap.
04180         if (const SCEVAddRecExpr *NewAR = dyn_cast<SCEVAddRecExpr>(S)) {
04181           const SCEVAddRecExpr *OldAR = cast<SCEVAddRecExpr>(LHS);
04182           const_cast<SCEVAddRecExpr *>(NewAR)->setNoWrapFlags(
04183             OldAR->getNoWrapFlags());
04184         }
04185         return S;
04186       }
04187     }
04188     break;
04189   case Instruction::Xor:
04190     if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
04191       // If the RHS of the xor is a signbit, then this is just an add.
04192       // Instcombine turns add of signbit into xor as a strength reduction step.
04193       if (CI->getValue().isSignBit())
04194         return getAddExpr(getSCEV(U->getOperand(0)),
04195                           getSCEV(U->getOperand(1)));
04196 
04197       // If the RHS of xor is -1, then this is a not operation.
04198       if (CI->isAllOnesValue())
04199         return getNotSCEV(getSCEV(U->getOperand(0)));
04200 
04201       // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
04202       // This is a variant of the check for xor with -1, and it handles
04203       // the case where instcombine has trimmed non-demanded bits out
04204       // of an xor with -1.
04205       if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U->getOperand(0)))
04206         if (ConstantInt *LCI = dyn_cast<ConstantInt>(BO->getOperand(1)))
04207           if (BO->getOpcode() == Instruction::And &&
04208               LCI->getValue() == CI->getValue())
04209             if (const SCEVZeroExtendExpr *Z =
04210                   dyn_cast<SCEVZeroExtendExpr>(getSCEV(U->getOperand(0)))) {
04211               Type *UTy = U->getType();
04212               const SCEV *Z0 = Z->getOperand();
04213               Type *Z0Ty = Z0->getType();
04214               unsigned Z0TySize = getTypeSizeInBits(Z0Ty);
04215 
04216               // If C is a low-bits mask, the zero extend is serving to
04217               // mask off the high bits. Complement the operand and
04218               // re-apply the zext.
04219               if (APIntOps::isMask(Z0TySize, CI->getValue()))
04220                 return getZeroExtendExpr(getNotSCEV(Z0), UTy);
04221 
04222               // If C is a single bit, it may be in the sign-bit position
04223               // before the zero-extend. In this case, represent the xor
04224               // using an add, which is equivalent, and re-apply the zext.
04225               APInt Trunc = CI->getValue().trunc(Z0TySize);
04226               if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() &&
04227                   Trunc.isSignBit())
04228                 return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)),
04229                                          UTy);
04230             }
04231     }
04232     break;
04233 
04234   case Instruction::Shl:
04235     // Turn shift left of a constant amount into a multiply.
04236     if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) {
04237       uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth();
04238 
04239       // If the shift count is not less than the bitwidth, the result of
04240       // the shift is undefined. Don't try to analyze it, because the
04241       // resolution chosen here may differ from the resolution chosen in
04242       // other parts of the compiler.
04243       if (SA->getValue().uge(BitWidth))
04244         break;
04245 
04246       Constant *X = ConstantInt::get(getContext(),
04247         APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
04248       return getMulExpr(getSCEV(U->getOperand(0)), getSCEV(X));
04249     }
04250     break;
04251 
04252   case Instruction::LShr:
04253     // Turn logical shift right of a constant into a unsigned divide.
04254     if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) {
04255       uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth();
04256 
04257       // If the shift count is not less than the bitwidth, the result of
04258       // the shift is undefined. Don't try to analyze it, because the
04259       // resolution chosen here may differ from the resolution chosen in
04260       // other parts of the compiler.
04261       if (SA->getValue().uge(BitWidth))
04262         break;
04263 
04264       Constant *X = ConstantInt::get(getContext(),
04265         APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
04266       return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(X));
04267     }
04268     break;
04269 
04270   case Instruction::AShr:
04271     // For a two-shift sext-inreg, use sext(trunc(x)) as the SCEV expression.
04272     if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1)))
04273       if (Operator *L = dyn_cast<Operator>(U->getOperand(0)))
04274         if (L->getOpcode() == Instruction::Shl &&
04275             L->getOperand(1) == U->getOperand(1)) {
04276           uint64_t BitWidth = getTypeSizeInBits(U->getType());
04277 
04278           // If the shift count is not less than the bitwidth, the result of
04279           // the shift is undefined. Don't try to analyze it, because the
04280           // resolution chosen here may differ from the resolution chosen in
04281           // other parts of the compiler.
04282           if (CI->getValue().uge(BitWidth))
04283             break;
04284 
04285           uint64_t Amt = BitWidth - CI->getZExtValue();
04286           if (Amt == BitWidth)
04287             return getSCEV(L->getOperand(0));       // shift by zero --> noop
04288           return
04289             getSignExtendExpr(getTruncateExpr(getSCEV(L->getOperand(0)),
04290                                               IntegerType::get(getContext(),
04291                                                                Amt)),
04292                               U->getType());
04293         }
04294     break;
04295 
04296   case Instruction::Trunc:
04297     return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType());
04298 
04299   case Instruction::ZExt:
04300     return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType());
04301 
04302   case Instruction::SExt:
04303     return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType());
04304 
04305   case Instruction::BitCast:
04306     // BitCasts are no-op casts so we just eliminate the cast.
04307     if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType()))
04308       return getSCEV(U->getOperand(0));
04309     break;
04310 
04311   // It's tempting to handle inttoptr and ptrtoint as no-ops, however this can
04312   // lead to pointer expressions which cannot safely be expanded to GEPs,
04313   // because ScalarEvolution doesn't respect the GEP aliasing rules when
04314   // simplifying integer expressions.
04315 
04316   case Instruction::GetElementPtr:
04317     return createNodeForGEP(cast<GEPOperator>(U));
04318 
04319   case Instruction::PHI:
04320     return createNodeForPHI(cast<PHINode>(U));
04321 
04322   case Instruction::Select:
04323     // This could be a smax or umax that was lowered earlier.
04324     // Try to recover it.
04325     if (ICmpInst *ICI = dyn_cast<ICmpInst>(U->getOperand(0))) {
04326       Value *LHS = ICI->getOperand(0);
04327       Value *RHS = ICI->getOperand(1);
04328       switch (ICI->getPredicate()) {
04329       case ICmpInst::ICMP_SLT:
04330       case ICmpInst::ICMP_SLE:
04331         std::swap(LHS, RHS);
04332         // fall through
04333       case ICmpInst::ICMP_SGT:
04334       case ICmpInst::ICMP_SGE:
04335         // a >s b ? a+x : b+x  ->  smax(a, b)+x
04336         // a >s b ? b+x : a+x  ->  smin(a, b)+x
04337         if (LHS->getType() == U->getType()) {
04338           const SCEV *LS = getSCEV(LHS);
04339           const SCEV *RS = getSCEV(RHS);
04340           const SCEV *LA = getSCEV(U->getOperand(1));
04341           const SCEV *RA = getSCEV(U->getOperand(2));
04342           const SCEV *LDiff = getMinusSCEV(LA, LS);
04343           const SCEV *RDiff = getMinusSCEV(RA, RS);
04344           if (LDiff == RDiff)
04345             return getAddExpr(getSMaxExpr(LS, RS), LDiff);
04346           LDiff = getMinusSCEV(LA, RS);
04347           RDiff = getMinusSCEV(RA, LS);
04348           if (LDiff == RDiff)
04349             return getAddExpr(getSMinExpr(LS, RS), LDiff);
04350         }
04351         break;
04352       case ICmpInst::ICMP_ULT:
04353       case ICmpInst::ICMP_ULE:
04354         std::swap(LHS, RHS);
04355         // fall through
04356       case ICmpInst::ICMP_UGT:
04357       case ICmpInst::ICMP_UGE:
04358         // a >u b ? a+x : b+x  ->  umax(a, b)+x
04359         // a >u b ? b+x : a+x  ->  umin(a, b)+x
04360         if (LHS->getType() == U->getType()) {
04361           const SCEV *LS = getSCEV(LHS);
04362           const SCEV *RS = getSCEV(RHS);
04363           const SCEV *LA = getSCEV(U->getOperand(1));
04364           const SCEV *RA = getSCEV(U->getOperand(2));
04365           const SCEV *LDiff = getMinusSCEV(LA, LS);
04366           const SCEV *RDiff = getMinusSCEV(RA, RS);
04367           if (LDiff == RDiff)
04368             return getAddExpr(getUMaxExpr(LS, RS), LDiff);
04369           LDiff = getMinusSCEV(LA, RS);
04370           RDiff = getMinusSCEV(RA, LS);
04371           if (LDiff == RDiff)
04372             return getAddExpr(getUMinExpr(LS, RS), LDiff);
04373         }
04374         break;
04375       case ICmpInst::ICMP_NE:
04376         // n != 0 ? n+x : 1+x  ->  umax(n, 1)+x
04377         if (LHS->getType() == U->getType() &&
04378             isa<ConstantInt>(RHS) &&
04379             cast<ConstantInt>(RHS)->isZero()) {
04380           const SCEV *One = getConstant(LHS->getType(), 1);
04381           const SCEV *LS = getSCEV(LHS);
04382           const SCEV *LA = getSCEV(U->getOperand(1));
04383           const SCEV *RA = getSCEV(U->getOperand(2));
04384           const SCEV *LDiff = getMinusSCEV(LA, LS);
04385           const SCEV *RDiff = getMinusSCEV(RA, One);
04386           if (LDiff == RDiff)
04387             return getAddExpr(getUMaxExpr(One, LS), LDiff);
04388         }
04389         break;
04390       case ICmpInst::ICMP_EQ:
04391         // n == 0 ? 1+x : n+x  ->  umax(n, 1)+x
04392         if (LHS->getType() == U->getType() &&
04393             isa<ConstantInt>(RHS) &&
04394             cast<ConstantInt>(RHS)->isZero()) {
04395           const SCEV *One = getConstant(LHS->getType(), 1);
04396           const SCEV *LS = getSCEV(LHS);
04397           const SCEV *LA = getSCEV(U->getOperand(1));
04398           const SCEV *RA = getSCEV(U->getOperand(2));
04399           const SCEV *LDiff = getMinusSCEV(LA, One);
04400           const SCEV *RDiff = getMinusSCEV(RA, LS);
04401           if (LDiff == RDiff)
04402             return getAddExpr(getUMaxExpr(One, LS), LDiff);
04403         }
04404         break;
04405       default:
04406         break;
04407       }
04408     }
04409 
04410   default: // We cannot analyze this expression.
04411     break;
04412   }
04413 
04414   return getUnknown(V);
04415 }
04416 
04417 
04418 
04419 //===----------------------------------------------------------------------===//
04420 //                   Iteration Count Computation Code
04421 //
04422 
04423 unsigned ScalarEvolution::getSmallConstantTripCount(Loop *L) {
04424   if (BasicBlock *ExitingBB = L->getExitingBlock())
04425     return getSmallConstantTripCount(L, ExitingBB);
04426 
04427   // No trip count information for multiple exits.
04428   return 0;
04429 }
04430 
04431 /// getSmallConstantTripCount - Returns the maximum trip count of this loop as a
04432 /// normal unsigned value. Returns 0 if the trip count is unknown or not
04433 /// constant. Will also return 0 if the maximum trip count is very large (>=
04434 /// 2^32).
04435 ///
04436 /// This "trip count" assumes that control exits via ExitingBlock. More
04437 /// precisely, it is the number of times that control may reach ExitingBlock
04438 /// before taking the branch. For loops with multiple exits, it may not be the
04439 /// number times that the loop header executes because the loop may exit
04440 /// prematurely via another branch.
04441 unsigned ScalarEvolution::getSmallConstantTripCount(Loop *L,
04442                                                     BasicBlock *ExitingBlock) {
04443   assert(ExitingBlock && "Must pass a non-null exiting block!");
04444   assert(L->isLoopExiting(ExitingBlock) &&
04445          "Exiting block must actually branch out of the loop!");
04446   const SCEVConstant *ExitCount =
04447       dyn_cast<SCEVConstant>(getExitCount(L, ExitingBlock));
04448   if (!ExitCount)
04449     return 0;
04450 
04451   ConstantInt *ExitConst = ExitCount->getValue();
04452 
04453   // Guard against huge trip counts.
04454   if (ExitConst->getValue().getActiveBits() > 32)
04455     return 0;
04456 
04457   // In case of integer overflow, this returns 0, which is correct.
04458   return ((unsigned)ExitConst->getZExtValue()) + 1;
04459 }
04460 
04461 unsigned ScalarEvolution::getSmallConstantTripMultiple(Loop *L) {
04462   if (BasicBlock *ExitingBB = L->getExitingBlock())
04463     return getSmallConstantTripMultiple(L, ExitingBB);
04464 
04465   // No trip multiple information for multiple exits.
04466   return 0;
04467 }
04468 
04469 /// getSmallConstantTripMultiple - Returns the largest constant divisor of the
04470 /// trip count of this loop as a normal unsigned value, if possible. This
04471 /// means that the actual trip count is always a multiple of the returned
04472 /// value (don't forget the trip count could very well be zero as well!).
04473 ///
04474 /// Returns 1 if the trip count is unknown or not guaranteed to be the
04475 /// multiple of a constant (which is also the case if the trip count is simply
04476 /// constant, use getSmallConstantTripCount for that case), Will also return 1
04477 /// if the trip count is very large (>= 2^32).
04478 ///
04479 /// As explained in the comments for getSmallConstantTripCount, this assumes
04480 /// that control exits the loop via ExitingBlock.
04481 unsigned
04482 ScalarEvolution::getSmallConstantTripMultiple(Loop *L,
04483                                               BasicBlock *ExitingBlock) {
04484   assert(ExitingBlock && "Must pass a non-null exiting block!");
04485   assert(L->isLoopExiting(ExitingBlock) &&
04486          "Exiting block must actually branch out of the loop!");
04487   const SCEV *ExitCount = getExitCount(L, ExitingBlock);
04488   if (ExitCount == getCouldNotCompute())
04489     return 1;
04490 
04491   // Get the trip count from the BE count by adding 1.
04492   const SCEV *TCMul = getAddExpr(ExitCount,
04493                                  getConstant(ExitCount->getType(), 1));
04494   // FIXME: SCEV distributes multiplication as V1*C1 + V2*C1. We could attempt
04495   // to factor simple cases.
04496   if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(TCMul))
04497     TCMul = Mul->getOperand(0);
04498 
04499   const SCEVConstant *MulC = dyn_cast<SCEVConstant>(TCMul);
04500   if (!MulC)
04501     return 1;
04502 
04503   ConstantInt *Result = MulC->getValue();
04504 
04505   // Guard against huge trip counts (this requires checking
04506   // for zero to handle the case where the trip count == -1 and the
04507   // addition wraps).
04508   if (!Result || Result->getValue().getActiveBits() > 32 ||
04509       Result->getValue().getActiveBits() == 0)
04510     return 1;
04511 
04512   return (unsigned)Result->getZExtValue();
04513 }
04514 
04515 // getExitCount - Get the expression for the number of loop iterations for which
04516 // this loop is guaranteed not to exit via ExitingBlock. Otherwise return
04517 // SCEVCouldNotCompute.
04518 const SCEV *ScalarEvolution::getExitCount(Loop *L, BasicBlock *ExitingBlock) {
04519   return getBackedgeTakenInfo(L).getExact(ExitingBlock, this);
04520 }
04521 
04522 /// getBackedgeTakenCount - If the specified loop has a predictable
04523 /// backedge-taken count, return it, otherwise return a SCEVCouldNotCompute
04524 /// object. The backedge-taken count is the number of times the loop header
04525 /// will be branched to from within the loop. This is one less than the
04526 /// trip count of the loop, since it doesn't count the first iteration,
04527 /// when the header is branched to from outside the loop.
04528 ///
04529 /// Note that it is not valid to call this method on a loop without a
04530 /// loop-invariant backedge-taken count (see
04531 /// hasLoopInvariantBackedgeTakenCount).
04532 ///
04533 const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L) {
04534   return getBackedgeTakenInfo(L).getExact(this);
04535 }
04536 
04537 /// getMaxBackedgeTakenCount - Similar to getBackedgeTakenCount, except
04538 /// return the least SCEV value that is known never to be less than the
04539 /// actual backedge taken count.
04540 const SCEV *ScalarEvolution::getMaxBackedgeTakenCount(const Loop *L) {
04541   return getBackedgeTakenInfo(L).getMax(this);
04542 }
04543 
04544 /// PushLoopPHIs - Push PHI nodes in the header of the given loop
04545 /// onto the given Worklist.
04546 static void
04547 PushLoopPHIs(const Loop *L, SmallVectorImpl<Instruction *> &Worklist) {
04548   BasicBlock *Header = L->getHeader();
04549 
04550   // Push all Loop-header PHIs onto the Worklist stack.
04551   for (BasicBlock::iterator I = Header->begin();
04552        PHINode *PN = dyn_cast<PHINode>(I); ++I)
04553     Worklist.push_back(PN);
04554 }
04555 
04556 const ScalarEvolution::BackedgeTakenInfo &
04557 ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
04558   // Initially insert an invalid entry for this loop. If the insertion
04559   // succeeds, proceed to actually compute a backedge-taken count and
04560   // update the value. The temporary CouldNotCompute value tells SCEV
04561   // code elsewhere that it shouldn't attempt to request a new
04562   // backedge-taken count, which could result in infinite recursion.
04563   std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
04564     BackedgeTakenCounts.insert(std::make_pair(L, BackedgeTakenInfo()));
04565   if (!Pair.second)
04566     return Pair.first->second;
04567 
04568   // ComputeBackedgeTakenCount may allocate memory for its result. Inserting it
04569   // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
04570   // must be cleared in this scope.
04571   BackedgeTakenInfo Result = ComputeBackedgeTakenCount(L);
04572 
04573   if (Result.getExact(this) != getCouldNotCompute()) {
04574     assert(isLoopInvariant(Result.getExact(this), L) &&
04575            isLoopInvariant(Result.getMax(this), L) &&
04576            "Computed backedge-taken count isn't loop invariant for loop!");
04577     ++NumTripCountsComputed;
04578   }
04579   else if (Result.getMax(this) == getCouldNotCompute() &&
04580            isa<PHINode>(L->getHeader()->begin())) {
04581     // Only count loops that have phi nodes as not being computable.
04582     ++NumTripCountsNotComputed;
04583   }
04584 
04585   // Now that we know more about the trip count for this loop, forget any
04586   // existing SCEV values for PHI nodes in this loop since they are only
04587   // conservative estimates made without the benefit of trip count
04588   // information. This is similar to the code in forgetLoop, except that
04589   // it handles SCEVUnknown PHI nodes specially.
04590   if (Result.hasAnyInfo()) {
04591     SmallVector<Instruction *, 16> Worklist;
04592     PushLoopPHIs(L, Worklist);
04593 
04594     SmallPtrSet<Instruction *, 8> Visited;
04595     while (!Worklist.empty()) {
04596       Instruction *I = Worklist.pop_back_val();
04597       if (!Visited.insert(I).second)
04598         continue;
04599 
04600       ValueExprMapType::iterator It =
04601         ValueExprMap.find_as(static_cast<Value *>(I));
04602       if (It != ValueExprMap.end()) {
04603         const SCEV *Old = It->second;
04604 
04605         // SCEVUnknown for a PHI either means that it has an unrecognized
04606         // structure, or it's a PHI that's in the progress of being computed
04607         // by createNodeForPHI.  In the former case, additional loop trip
04608         // count information isn't going to change anything. In the later
04609         // case, createNodeForPHI will perform the necessary updates on its
04610         // own when it gets to that point.
04611         if (!isa<PHINode>(I) || !isa<SCEVUnknown>(Old)) {
04612           forgetMemoizedResults(Old);
04613           ValueExprMap.erase(It);
04614         }
04615         if (PHINode *PN = dyn_cast<PHINode>(I))
04616           ConstantEvolutionLoopExitValue.erase(PN);
04617       }
04618 
04619       PushDefUseChildren(I, Worklist);
04620     }
04621   }
04622 
04623   // Re-lookup the insert position, since the call to
04624   // ComputeBackedgeTakenCount above could result in a
04625   // recusive call to getBackedgeTakenInfo (on a different
04626   // loop), which would invalidate the iterator computed
04627   // earlier.
04628   return BackedgeTakenCounts.find(L)->second = Result;
04629 }
04630 
04631 /// forgetLoop - This method should be called by the client when it has
04632 /// changed a loop in a way that may effect ScalarEvolution's ability to
04633 /// compute a trip count, or if the loop is deleted.
04634 void ScalarEvolution::forgetLoop(const Loop *L) {
04635   // Drop any stored trip count value.
04636   DenseMap<const Loop*, BackedgeTakenInfo>::iterator BTCPos =
04637     BackedgeTakenCounts.find(L);
04638   if (BTCPos != BackedgeTakenCounts.end()) {
04639     BTCPos->second.clear();
04640     BackedgeTakenCounts.erase(BTCPos);
04641   }
04642 
04643   // Drop information about expressions based on loop-header PHIs.
04644   SmallVector<Instruction *, 16> Worklist;
04645   PushLoopPHIs(L, Worklist);
04646 
04647   SmallPtrSet<Instruction *, 8> Visited;
04648   while (!Worklist.empty()) {
04649     Instruction *I = Worklist.pop_back_val();
04650     if (!Visited.insert(I).second)
04651       continue;
04652 
04653     ValueExprMapType::iterator It =
04654       ValueExprMap.find_as(static_cast<Value *>(I));
04655     if (It != ValueExprMap.end()) {
04656       forgetMemoizedResults(It->second);
04657       ValueExprMap.erase(It);
04658       if (PHINode *PN = dyn_cast<PHINode>(I))
04659         ConstantEvolutionLoopExitValue.erase(PN);
04660     }
04661 
04662     PushDefUseChildren(I, Worklist);
04663   }
04664 
04665   // Forget all contained loops too, to avoid dangling entries in the
04666   // ValuesAtScopes map.
04667   for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I)
04668     forgetLoop(*I);
04669 }
04670 
04671 /// forgetValue - This method should be called by the client when it has
04672 /// changed a value in a way that may effect its value, or which may
04673 /// disconnect it from a def-use chain linking it to a loop.
04674 void ScalarEvolution::forgetValue(Value *V) {
04675   Instruction *I = dyn_cast<Instruction>(V);
04676   if (!I) return;
04677 
04678   // Drop information about expressions based on loop-header PHIs.
04679   SmallVector<Instruction *, 16> Worklist;
04680   Worklist.push_back(I);
04681 
04682   SmallPtrSet<Instruction *, 8> Visited;
04683   while (!Worklist.empty()) {
04684     I = Worklist.pop_back_val();
04685     if (!Visited.insert(I).second)
04686       continue;
04687 
04688     ValueExprMapType::iterator It =
04689       ValueExprMap.find_as(static_cast<Value *>(I));
04690     if (It != ValueExprMap.end()) {
04691       forgetMemoizedResults(It->second);
04692       ValueExprMap.erase(It);
04693       if (PHINode *PN = dyn_cast<PHINode>(I))
04694         ConstantEvolutionLoopExitValue.erase(PN);
04695     }
04696 
04697     PushDefUseChildren(I, Worklist);
04698   }
04699 }
04700 
04701 /// getExact - Get the exact loop backedge taken count considering all loop
04702 /// exits. A computable result can only be return for loops with a single exit.
04703 /// Returning the minimum taken count among all exits is incorrect because one
04704 /// of the loop's exit limit's may have been skipped. HowFarToZero assumes that
04705 /// the limit of each loop test is never skipped. This is a valid assumption as
04706 /// long as the loop exits via that test. For precise results, it is the
04707 /// caller's responsibility to specify the relevant loop exit using
04708 /// getExact(ExitingBlock, SE).
04709 const SCEV *
04710 ScalarEvolution::BackedgeTakenInfo::getExact(ScalarEvolution *SE) const {
04711   // If any exits were not computable, the loop is not computable.
04712   if (!ExitNotTaken.isCompleteList()) return SE->getCouldNotCompute();
04713 
04714   // We need exactly one computable exit.
04715   if (!ExitNotTaken.ExitingBlock) return SE->getCouldNotCompute();
04716   assert(ExitNotTaken.ExactNotTaken && "uninitialized not-taken info");
04717 
04718   const SCEV *BECount = nullptr;
04719   for (const ExitNotTakenInfo *ENT = &ExitNotTaken;
04720        ENT != nullptr; ENT = ENT->getNextExit()) {
04721 
04722     assert(ENT->ExactNotTaken != SE->getCouldNotCompute() && "bad exit SCEV");
04723 
04724     if (!BECount)
04725       BECount = ENT->ExactNotTaken;
04726     else if (BECount != ENT->ExactNotTaken)
04727       return SE->getCouldNotCompute();
04728   }
04729   assert(BECount && "Invalid not taken count for loop exit");
04730   return BECount;
04731 }
04732 
04733 /// getExact - Get the exact not taken count for this loop exit.
04734 const SCEV *
04735 ScalarEvolution::BackedgeTakenInfo::getExact(BasicBlock *ExitingBlock,
04736                                              ScalarEvolution *SE) const {
04737   for (const ExitNotTakenInfo *ENT = &ExitNotTaken;
04738        ENT != nullptr; ENT = ENT->getNextExit()) {
04739 
04740     if (ENT->ExitingBlock == ExitingBlock)
04741       return ENT->ExactNotTaken;
04742   }
04743   return SE->getCouldNotCompute();
04744 }
04745 
04746 /// getMax - Get the max backedge taken count for the loop.
04747 const SCEV *
04748 ScalarEvolution::BackedgeTakenInfo::getMax(ScalarEvolution *SE) const {
04749   return Max ? Max : SE->getCouldNotCompute();
04750 }
04751 
04752 bool ScalarEvolution::BackedgeTakenInfo::hasOperand(const SCEV *S,
04753                                                     ScalarEvolution *SE) const {
04754   if (Max && Max != SE->getCouldNotCompute() && SE->hasOperand(Max, S))
04755     return true;
04756 
04757   if (!ExitNotTaken.ExitingBlock)
04758     return false;
04759 
04760   for (const ExitNotTakenInfo *ENT = &ExitNotTaken;
04761        ENT != nullptr; ENT = ENT->getNextExit()) {
04762 
04763     if (ENT->ExactNotTaken != SE->getCouldNotCompute()
04764         && SE->hasOperand(ENT->ExactNotTaken, S)) {
04765       return true;
04766     }
04767   }
04768   return false;
04769 }
04770 
04771 /// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
04772 /// computable exit into a persistent ExitNotTakenInfo array.
04773 ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
04774   SmallVectorImpl< std::pair<BasicBlock *, const SCEV *> > &ExitCounts,
04775   bool Complete, const SCEV *MaxCount) : Max(MaxCount) {
04776 
04777   if (!Complete)
04778     ExitNotTaken.setIncomplete();
04779 
04780   unsigned NumExits = ExitCounts.size();
04781   if (NumExits == 0) return;
04782 
04783   ExitNotTaken.ExitingBlock = ExitCounts[0].first;
04784   ExitNotTaken.ExactNotTaken = ExitCounts[0].second;
04785   if (NumExits == 1) return;
04786 
04787   // Handle the rare case of multiple computable exits.
04788   ExitNotTakenInfo *ENT = new ExitNotTakenInfo[NumExits-1];
04789 
04790   ExitNotTakenInfo *PrevENT = &ExitNotTaken;
04791   for (unsigned i = 1; i < NumExits; ++i, PrevENT = ENT, ++ENT) {
04792     PrevENT->setNextExit(ENT);
04793     ENT->ExitingBlock = ExitCounts[i].first;
04794     ENT->ExactNotTaken = ExitCounts[i].second;
04795   }
04796 }
04797 
04798 /// clear - Invalidate this result and free the ExitNotTakenInfo array.
04799 void ScalarEvolution::BackedgeTakenInfo::clear() {
04800   ExitNotTaken.ExitingBlock = nullptr;
04801   ExitNotTaken.ExactNotTaken = nullptr;
04802   delete[] ExitNotTaken.getNextExit();
04803 }
04804 
04805 /// ComputeBackedgeTakenCount - Compute the number of times the backedge
04806 /// of the specified loop will execute.
04807 ScalarEvolution::BackedgeTakenInfo
04808 ScalarEvolution::ComputeBackedgeTakenCount(const Loop *L) {
04809   SmallVector<BasicBlock *, 8> ExitingBlocks;
04810   L->getExitingBlocks(ExitingBlocks);
04811 
04812   SmallVector<std::pair<BasicBlock *, const SCEV *>, 4> ExitCounts;
04813   bool CouldComputeBECount = true;
04814   BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
04815   const SCEV *MustExitMaxBECount = nullptr;
04816   const SCEV *MayExitMaxBECount = nullptr;
04817 
04818   // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
04819   // and compute maxBECount.
04820   for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) {
04821     BasicBlock *ExitBB = ExitingBlocks[i];
04822     ExitLimit EL = ComputeExitLimit(L, ExitBB);
04823 
04824     // 1. For each exit that can be computed, add an entry to ExitCounts.
04825     // CouldComputeBECount is true only if all exits can be computed.
04826     if (EL.Exact == getCouldNotCompute())
04827       // We couldn't compute an exact value for this exit, so
04828       // we won't be able to compute an exact value for the loop.
04829       CouldComputeBECount = false;
04830     else
04831       ExitCounts.push_back(std::make_pair(ExitBB, EL.Exact));
04832 
04833     // 2. Derive the loop's MaxBECount from each exit's max number of
04834     // non-exiting iterations. Partition the loop exits into two kinds:
04835     // LoopMustExits and LoopMayExits.
04836     //
04837     // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
04838     // is a LoopMayExit.  If any computable LoopMustExit is found, then
04839     // MaxBECount is the minimum EL.Max of computable LoopMustExits. Otherwise,
04840     // MaxBECount is conservatively the maximum EL.Max, where CouldNotCompute is
04841     // considered greater than any computable EL.Max.
04842     if (EL.Max != getCouldNotCompute() && Latch &&
04843         DT->dominates(ExitBB, Latch)) {
04844       if (!MustExitMaxBECount)
04845         MustExitMaxBECount = EL.Max;
04846       else {
04847         MustExitMaxBECount =
04848           getUMinFromMismatchedTypes(MustExitMaxBECount, EL.Max);
04849       }
04850     } else if (MayExitMaxBECount != getCouldNotCompute()) {
04851       if (!MayExitMaxBECount || EL.Max == getCouldNotCompute())
04852         MayExitMaxBECount = EL.Max;
04853       else {
04854         MayExitMaxBECount =
04855           getUMaxFromMismatchedTypes(MayExitMaxBECount, EL.Max);
04856       }
04857     }
04858   }
04859   const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
04860     (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
04861   return BackedgeTakenInfo(ExitCounts, CouldComputeBECount, MaxBECount);
04862 }
04863 
04864 /// ComputeExitLimit - Compute the number of times the backedge of the specified
04865 /// loop will execute if it exits via the specified block.
04866 ScalarEvolution::ExitLimit
04867 ScalarEvolution::ComputeExitLimit(const Loop *L, BasicBlock *ExitingBlock) {
04868 
04869   // Okay, we've chosen an exiting block.  See what condition causes us to
04870   // exit at this block and remember the exit block and whether all other targets
04871   // lead to the loop header.
04872   bool MustExecuteLoopHeader = true;
04873   BasicBlock *Exit = nullptr;
04874   for (succ_iterator SI = succ_begin(ExitingBlock), SE = succ_end(ExitingBlock);
04875        SI != SE; ++SI)
04876     if (!L->contains(*SI)) {
04877       if (Exit) // Multiple exit successors.
04878         return getCouldNotCompute();
04879       Exit = *SI;
04880     } else if (*SI != L->getHeader()) {
04881       MustExecuteLoopHeader = false;
04882     }
04883 
04884   // At this point, we know we have a conditional branch that determines whether
04885   // the loop is exited.  However, we don't know if the branch is executed each
04886   // time through the loop.  If not, then the execution count of the branch will
04887   // not be equal to the trip count of the loop.
04888   //
04889   // Currently we check for this by checking to see if the Exit branch goes to
04890   // the loop header.  If so, we know it will always execute the same number of
04891   // times as the loop.  We also handle the case where the exit block *is* the
04892   // loop header.  This is common for un-rotated loops.
04893   //
04894   // If both of those tests fail, walk up the unique predecessor chain to the
04895   // header, stopping if there is an edge that doesn't exit the loop. If the
04896   // header is reached, the execution count of the branch will be equal to the
04897   // trip count of the loop.
04898   //
04899   //  More extensive analysis could be done to handle more cases here.
04900   //
04901   if (!MustExecuteLoopHeader && ExitingBlock != L->getHeader()) {
04902     // The simple checks failed, try climbing the unique predecessor chain
04903     // up to the header.
04904     bool Ok = false;
04905     for (BasicBlock *BB = ExitingBlock; BB; ) {
04906       BasicBlock *Pred = BB->getUniquePredecessor();
04907       if (!Pred)
04908         return getCouldNotCompute();
04909       TerminatorInst *PredTerm = Pred->getTerminator();
04910       for (unsigned i = 0, e = PredTerm->getNumSuccessors(); i != e; ++i) {
04911         BasicBlock *PredSucc = PredTerm->getSuccessor(i);
04912         if (PredSucc == BB)
04913           continue;
04914         // If the predecessor has a successor that isn't BB and isn't
04915         // outside the loop, assume the worst.
04916         if (L->contains(PredSucc))
04917           return getCouldNotCompute();
04918       }
04919       if (Pred == L->getHeader()) {
04920         Ok = true;
04921         break;
04922       }
04923       BB = Pred;
04924     }
04925     if (!Ok)
04926       return getCouldNotCompute();
04927   }
04928 
04929   bool IsOnlyExit = (L->getExitingBlock() != nullptr);
04930   TerminatorInst *Term = ExitingBlock->getTerminator();
04931   if (BranchInst *BI = dyn_cast<BranchInst>(Term)) {
04932     assert(BI->isConditional() && "If unconditional, it can't be in loop!");
04933     // Proceed to the next level to examine the exit condition expression.
04934     return ComputeExitLimitFromCond(L, BI->getCondition(), BI->getSuccessor(0),
04935                                     BI->getSuccessor(1),
04936                                     /*ControlsExit=*/IsOnlyExit);
04937   }
04938 
04939   if (SwitchInst *SI = dyn_cast<SwitchInst>(Term))
04940     return ComputeExitLimitFromSingleExitSwitch(L, SI, Exit,
04941                                                 /*ControlsExit=*/IsOnlyExit);
04942 
04943   return getCouldNotCompute();
04944 }
04945 
04946 /// ComputeExitLimitFromCond - Compute the number of times the
04947 /// backedge of the specified loop will execute if its exit condition
04948 /// were a conditional branch of ExitCond, TBB, and FBB.
04949 ///
04950 /// @param ControlsExit is true if ExitCond directly controls the exit
04951 /// branch. In this case, we can assume that the loop exits only if the
04952 /// condition is true and can infer that failing to meet the condition prior to
04953 /// integer wraparound results in undefined behavior.
04954 ScalarEvolution::ExitLimit
04955 ScalarEvolution::ComputeExitLimitFromCond(const Loop *L,
04956                                           Value *ExitCond,
04957                                           BasicBlock *TBB,
04958                                           BasicBlock *FBB,
04959                                           bool ControlsExit) {
04960   // Check if the controlling expression for this loop is an And or Or.
04961   if (BinaryOperator *BO = dyn_cast<BinaryOperator>(ExitCond)) {
04962     if (BO->getOpcode() == Instruction::And) {
04963       // Recurse on the operands of the and.
04964       bool EitherMayExit = L->contains(TBB);
04965       ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB,
04966                                                ControlsExit && !EitherMayExit);
04967       ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB,
04968                                                ControlsExit && !EitherMayExit);
04969       const SCEV *BECount = getCouldNotCompute();
04970       const SCEV *MaxBECount = getCouldNotCompute();
04971       if (EitherMayExit) {
04972         // Both conditions must be true for the loop to continue executing.
04973         // Choose the less conservative count.
04974         if (EL0.Exact == getCouldNotCompute() ||
04975             EL1.Exact == getCouldNotCompute())
04976           BECount = getCouldNotCompute();
04977         else
04978           BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact);
04979         if (EL0.Max == getCouldNotCompute())
04980           MaxBECount = EL1.Max;
04981         else if (EL1.Max == getCouldNotCompute())
04982           MaxBECount = EL0.Max;
04983         else
04984           MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max);
04985       } else {
04986         // Both conditions must be true at the same time for the loop to exit.
04987         // For now, be conservative.
04988         assert(L->contains(FBB) && "Loop block has no successor in loop!");
04989         if (EL0.Max == EL1.Max)
04990           MaxBECount = EL0.Max;
04991         if (EL0.Exact == EL1.Exact)
04992           BECount = EL0.Exact;
04993       }
04994 
04995       return ExitLimit(BECount, MaxBECount);
04996     }
04997     if (BO->getOpcode() == Instruction::Or) {
04998       // Recurse on the operands of the or.
04999       bool EitherMayExit = L->contains(FBB);
05000       ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB,
05001                                                ControlsExit && !EitherMayExit);
05002       ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB,
05003                                                ControlsExit && !EitherMayExit);
05004       const SCEV *BECount = getCouldNotCompute();
05005       const SCEV *MaxBECount = getCouldNotCompute();
05006       if (EitherMayExit) {
05007         // Both conditions must be false for the loop to continue executing.
05008         // Choose the less conservative count.
05009         if (EL0.Exact == getCouldNotCompute() ||
05010             EL1.Exact == getCouldNotCompute())
05011           BECount = getCouldNotCompute();
05012         else
05013           BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact);
05014         if (EL0.Max == getCouldNotCompute())
05015           MaxBECount = EL1.Max;
05016         else if (EL1.Max == getCouldNotCompute())
05017           MaxBECount = EL0.Max;
05018         else
05019           MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max);
05020       } else {
05021         // Both conditions must be false at the same time for the loop to exit.
05022         // For now, be conservative.
05023         assert(L->contains(TBB) && "Loop block has no successor in loop!");
05024         if (EL0.Max == EL1.Max)
05025           MaxBECount = EL0.Max;
05026         if (EL0.Exact == EL1.Exact)
05027           BECount = EL0.Exact;
05028       }
05029 
05030       return ExitLimit(BECount, MaxBECount);
05031     }
05032   }
05033 
05034   // With an icmp, it may be feasible to compute an exact backedge-taken count.
05035   // Proceed to the next level to examine the icmp.
05036   if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond))
05037     return ComputeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, ControlsExit);
05038 
05039   // Check for a constant condition. These are normally stripped out by
05040   // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
05041   // preserve the CFG and is temporarily leaving constant conditions
05042   // in place.
05043   if (ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) {
05044     if (L->contains(FBB) == !CI->getZExtValue())
05045       // The backedge is always taken.
05046       return getCouldNotCompute();
05047     else
05048       // The backedge is never taken.
05049       return getConstant(CI->getType(), 0);
05050   }
05051 
05052   // If it's not an integer or pointer comparison then compute it the hard way.
05053   return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB));
05054 }
05055 
05056 /// ComputeExitLimitFromICmp - Compute the number of times the
05057 /// backedge of the specified loop will execute if its exit condition
05058 /// were a conditional branch of the ICmpInst ExitCond, TBB, and FBB.
05059 ScalarEvolution::ExitLimit
05060 ScalarEvolution::ComputeExitLimitFromICmp(const Loop *L,
05061                                           ICmpInst *ExitCond,
05062                                           BasicBlock *TBB,
05063                                           BasicBlock *FBB,
05064                                           bool ControlsExit) {
05065 
05066   // If the condition was exit on true, convert the condition to exit on false
05067   ICmpInst::Predicate Cond;
05068   if (!L->contains(FBB))
05069     Cond = ExitCond->getPredicate();
05070   else
05071     Cond = ExitCond->getInversePredicate();
05072 
05073   // Handle common loops like: for (X = "string"; *X; ++X)
05074   if (LoadInst *LI = dyn_cast<LoadInst>(ExitCond->getOperand(0)))
05075     if (Constant *RHS = dyn_cast<Constant>(ExitCond->getOperand(1))) {
05076       ExitLimit ItCnt =
05077         ComputeLoadConstantCompareExitLimit(LI, RHS, L, Cond);
05078       if (ItCnt.hasAnyInfo())
05079         return ItCnt;
05080     }
05081 
05082   const SCEV *LHS = getSCEV(ExitCond->getOperand(0));
05083   const SCEV *RHS = getSCEV(ExitCond->getOperand(1));
05084 
05085   // Try to evaluate any dependencies out of the loop.
05086   LHS = getSCEVAtScope(LHS, L);
05087   RHS = getSCEVAtScope(RHS, L);
05088 
05089   // At this point, we would like to compute how many iterations of the
05090   // loop the predicate will return true for these inputs.
05091   if (isLoopInvariant(LHS, L) && !isLoopInvariant(RHS, L)) {
05092     // If there is a loop-invariant, force it into the RHS.
05093     std::swap(LHS, RHS);
05094     Cond = ICmpInst::getSwappedPredicate(Cond);
05095   }
05096 
05097   // Simplify the operands before analyzing them.
05098   (void)SimplifyICmpOperands(Cond, LHS, RHS);
05099 
05100   // If we have a comparison of a chrec against a constant, try to use value
05101   // ranges to answer this query.
05102   if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS))
05103     if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS))
05104       if (AddRec->getLoop() == L) {
05105         // Form the constant range.
05106         ConstantRange CompRange(
05107             ICmpInst::makeConstantRange(Cond, RHSC->getValue()->getValue()));
05108 
05109         const SCEV *Ret = AddRec->getNumIterationsInRange(CompRange, *this);
05110         if (!isa<SCEVCouldNotCompute>(Ret)) return Ret;
05111       }
05112 
05113   switch (Cond) {
05114   case ICmpInst::ICMP_NE: {                     // while (X != Y)
05115     // Convert to: while (X-Y != 0)
05116     ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit);
05117     if (EL.hasAnyInfo()) return EL;
05118     break;
05119   }
05120   case ICmpInst::ICMP_EQ: {                     // while (X == Y)
05121     // Convert to: while (X-Y == 0)
05122     ExitLimit EL = HowFarToNonZero(getMinusSCEV(LHS, RHS), L);
05123     if (EL.hasAnyInfo()) return EL;
05124     break;
05125   }
05126   case ICmpInst::ICMP_SLT:
05127   case ICmpInst::ICMP_ULT: {                    // while (X < Y)
05128     bool IsSigned = Cond == ICmpInst::ICMP_SLT;
05129     ExitLimit EL = HowManyLessThans(LHS, RHS, L, IsSigned, ControlsExit);
05130     if (EL.hasAnyInfo()) return EL;
05131     break;
05132   }
05133   case ICmpInst::ICMP_SGT:
05134   case ICmpInst::ICMP_UGT: {                    // while (X > Y)
05135     bool IsSigned = Cond == ICmpInst::ICMP_SGT;
05136     ExitLimit EL = HowManyGreaterThans(LHS, RHS, L, IsSigned, ControlsExit);
05137     if (EL.hasAnyInfo()) return EL;
05138     break;
05139   }
05140   default:
05141 #if 0
05142     dbgs() << "ComputeBackedgeTakenCount ";
05143     if (ExitCond->getOperand(0)->getType()->isUnsigned())
05144       dbgs() << "[unsigned] ";
05145     dbgs() << *LHS << "   "
05146          << Instruction::getOpcodeName(Instruction::ICmp)
05147          << "   " << *RHS << "\n";
05148 #endif
05149     break;
05150   }
05151   return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB));
05152 }
05153 
05154 ScalarEvolution::ExitLimit
05155 ScalarEvolution::ComputeExitLimitFromSingleExitSwitch(const Loop *L,
05156                                                       SwitchInst *Switch,
05157                                                       BasicBlock *ExitingBlock,
05158                                                       bool ControlsExit) {
05159   assert(!L->contains(ExitingBlock) && "Not an exiting block!");
05160 
05161   // Give up if the exit is the default dest of a switch.
05162   if (Switch->getDefaultDest() == ExitingBlock)
05163     return getCouldNotCompute();
05164 
05165   assert(L->contains(Switch->getDefaultDest()) &&
05166          "Default case must not exit the loop!");
05167   const SCEV *LHS = getSCEVAtScope(Switch->getCondition(), L);
05168   const SCEV *RHS = getConstant(Switch->findCaseDest(ExitingBlock));
05169 
05170   // while (X != Y) --> while (X-Y != 0)
05171   ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, ControlsExit);
05172   if (EL.hasAnyInfo())
05173     return EL;
05174 
05175   return getCouldNotCompute();
05176 }
05177 
05178 static ConstantInt *
05179 EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
05180                                 ScalarEvolution &SE) {
05181   const SCEV *InVal = SE.getConstant(C);
05182   const SCEV *Val = AddRec->evaluateAtIteration(InVal, SE);
05183   assert(isa<SCEVConstant>(Val) &&
05184          "Evaluation of SCEV at constant didn't fold correctly?");
05185   return cast<SCEVConstant>(Val)->getValue();
05186 }
05187 
05188 /// ComputeLoadConstantCompareExitLimit - Given an exit condition of
05189 /// 'icmp op load X, cst', try to see if we can compute the backedge
05190 /// execution count.
05191 ScalarEvolution::ExitLimit
05192 ScalarEvolution::ComputeLoadConstantCompareExitLimit(
05193   LoadInst *LI,
05194   Constant *RHS,
05195   const Loop *L,
05196   ICmpInst::Predicate predicate) {
05197 
05198   if (LI->isVolatile()) return getCouldNotCompute();
05199 
05200   // Check to see if the loaded pointer is a getelementptr of a global.
05201   // TODO: Use SCEV instead of manually grubbing with GEPs.
05202   GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0));
05203   if (!GEP) return getCouldNotCompute();
05204 
05205   // Make sure that it is really a constant global we are gepping, with an
05206   // initializer, and make sure the first IDX is really 0.
05207   GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0));
05208   if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() ||
05209       GEP->getNumOperands() < 3 || !isa<Constant>(GEP->getOperand(1)) ||
05210       !cast<Constant>(GEP->getOperand(1))->isNullValue())
05211     return getCouldNotCompute();
05212 
05213   // Okay, we allow one non-constant index into the GEP instruction.
05214   Value *VarIdx = nullptr;
05215   std::vector<Constant*> Indexes;
05216   unsigned VarIdxNum = 0;
05217   for (unsigned i = 2, e = GEP->getNumOperands(); i != e; ++i)
05218     if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) {
05219       Indexes.push_back(CI);
05220     } else if (!isa<ConstantInt>(GEP->getOperand(i))) {
05221       if (VarIdx) return getCouldNotCompute();  // Multiple non-constant idx's.
05222       VarIdx = GEP->getOperand(i);
05223       VarIdxNum = i-2;
05224       Indexes.push_back(nullptr);
05225     }
05226 
05227   // Loop-invariant loads may be a byproduct of loop optimization. Skip them.
05228   if (!VarIdx)
05229     return getCouldNotCompute();
05230 
05231   // Okay, we know we have a (load (gep GV, 0, X)) comparison with a constant.
05232   // Check to see if X is a loop variant variable value now.
05233   const SCEV *Idx = getSCEV(VarIdx);
05234   Idx = getSCEVAtScope(Idx, L);
05235 
05236   // We can only recognize very limited forms of loop index expressions, in
05237   // particular, only affine AddRec's like {C1,+,C2}.
05238   const SCEVAddRecExpr *IdxExpr = dyn_cast<SCEVAddRecExpr>(Idx);
05239   if (!IdxExpr || !IdxExpr->isAffine() || isLoopInvariant(IdxExpr, L) ||
05240       !isa<SCEVConstant>(IdxExpr->getOperand(0)) ||
05241       !isa<SCEVConstant>(IdxExpr->getOperand(1)))
05242     return getCouldNotCompute();
05243 
05244   unsigned MaxSteps = MaxBruteForceIterations;
05245   for (unsigned IterationNum = 0; IterationNum != MaxSteps; ++IterationNum) {
05246     ConstantInt *ItCst = ConstantInt::get(
05247                            cast<IntegerType>(IdxExpr->getType()), IterationNum);
05248     ConstantInt *Val = EvaluateConstantChrecAtConstant(IdxExpr, ItCst, *this);
05249 
05250     // Form the GEP offset.
05251     Indexes[VarIdxNum] = Val;
05252 
05253     Constant *Result = ConstantFoldLoadThroughGEPIndices(GV->getInitializer(),
05254                                                          Indexes);
05255     if (!Result) break;  // Cannot compute!
05256 
05257     // Evaluate the condition for this iteration.
05258     Result = ConstantExpr::getICmp(predicate, Result, RHS);
05259     if (!isa<ConstantInt>(Result)) break;  // Couldn't decide for sure
05260     if (cast<ConstantInt>(Result)->getValue().isMinValue()) {
05261 #if 0
05262       dbgs() << "\n***\n*** Computed loop count " << *ItCst
05263              << "\n*** From global " << *GV << "*** BB: " << *L->getHeader()
05264              << "***\n";
05265 #endif
05266       ++NumArrayLenItCounts;
05267       return getConstant(ItCst);   // Found terminating iteration!
05268     }
05269   }
05270   return getCouldNotCompute();
05271 }
05272 
05273 
05274 /// CanConstantFold - Return true if we can constant fold an instruction of the
05275 /// specified type, assuming that all operands were constants.
05276 static bool CanConstantFold(const Instruction *I) {
05277   if (isa<BinaryOperator>(I) || isa<CmpInst>(I) ||
05278       isa<SelectInst>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
05279       isa<LoadInst>(I))
05280     return true;
05281 
05282   if (const CallInst *CI = dyn_cast<CallInst>(I))
05283     if (const Function *F = CI->getCalledFunction())
05284       return canConstantFoldCallTo(F);
05285   return false;
05286 }
05287 
05288 /// Determine whether this instruction can constant evolve within this loop
05289 /// assuming its operands can all constant evolve.
05290 static bool canConstantEvolve(Instruction *I, const Loop *L) {
05291   // An instruction outside of the loop can't be derived from a loop PHI.
05292   if (!L->contains(I)) return false;
05293 
05294   if (isa<PHINode>(I)) {
05295     if (L->getHeader() == I->getParent())
05296       return true;
05297     else
05298       // We don't currently keep track of the control flow needed to evaluate
05299       // PHIs, so we cannot handle PHIs inside of loops.
05300       return false;
05301   }
05302 
05303   // If we won't be able to constant fold this expression even if the operands
05304   // are constants, bail early.
05305   return CanConstantFold(I);
05306 }
05307 
05308 /// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
05309 /// recursing through each instruction operand until reaching a loop header phi.
05310 static PHINode *
05311 getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L,
05312                                DenseMap<Instruction *, PHINode *> &PHIMap) {
05313 
05314   // Otherwise, we can evaluate this instruction if all of its operands are
05315   // constant or derived from a PHI node themselves.
05316   PHINode *PHI = nullptr;
05317   for (Instruction::op_iterator OpI = UseInst->op_begin(),
05318          OpE = UseInst->op_end(); OpI != OpE; ++OpI) {
05319 
05320     if (isa<Constant>(*OpI)) continue;
05321 
05322     Instruction *OpInst = dyn_cast<Instruction>(*OpI);
05323     if (!OpInst || !canConstantEvolve(OpInst, L)) return nullptr;
05324 
05325     PHINode *P = dyn_cast<PHINode>(OpInst);
05326     if (!P)
05327       // If this operand is already visited, reuse the prior result.
05328       // We may have P != PHI if this is the deepest point at which the
05329       // inconsistent paths meet.
05330       P = PHIMap.lookup(OpInst);
05331     if (!P) {
05332       // Recurse and memoize the results, whether a phi is found or not.
05333       // This recursive call invalidates pointers into PHIMap.
05334       P = getConstantEvolvingPHIOperands(OpInst, L, PHIMap);
05335       PHIMap[OpInst] = P;
05336     }
05337     if (!P)
05338       return nullptr;  // Not evolving from PHI
05339     if (PHI && PHI != P)
05340       return nullptr;  // Evolving from multiple different PHIs.
05341     PHI = P;
05342   }
05343   // This is a expression evolving from a constant PHI!
05344   return PHI;
05345 }
05346 
05347 /// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
05348 /// in the loop that V is derived from.  We allow arbitrary operations along the
05349 /// way, but the operands of an operation must either be constants or a value
05350 /// derived from a constant PHI.  If this expression does not fit with these
05351 /// constraints, return null.
05352 static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) {
05353   Instruction *I = dyn_cast<Instruction>(V);
05354   if (!I || !canConstantEvolve(I, L)) return nullptr;
05355 
05356   if (PHINode *PN = dyn_cast<PHINode>(I)) {
05357     return PN;
05358   }
05359 
05360   // Record non-constant instructions contained by the loop.
05361   DenseMap<Instruction *, PHINode *> PHIMap;
05362   return getConstantEvolvingPHIOperands(I, L, PHIMap);
05363 }
05364 
05365 /// EvaluateExpression - Given an expression that passes the
05366 /// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
05367 /// in the loop has the value PHIVal.  If we can't fold this expression for some
05368 /// reason, return null.
05369 static Constant *EvaluateExpression(Value *V, const Loop *L,
05370                                     DenseMap<Instruction *, Constant *> &Vals,
05371                                     const DataLayout *DL,
05372                                     const TargetLibraryInfo *TLI) {
05373   // Convenient constant check, but redundant for recursive calls.
05374   if (Constant *C = dyn_cast<Constant>(V)) return C;
05375   Instruction *I = dyn_cast<Instruction>(V);
05376   if (!I) return nullptr;
05377 
05378   if (Constant *C = Vals.lookup(I)) return C;
05379 
05380   // An instruction inside the loop depends on a value outside the loop that we
05381   // weren't given a mapping for, or a value such as a call inside the loop.
05382   if (!canConstantEvolve(I, L)) return nullptr;
05383 
05384   // An unmapped PHI can be due to a branch or another loop inside this loop,
05385   // or due to this not being the initial iteration through a loop where we
05386   // couldn't compute the evolution of this particular PHI last time.
05387   if (isa<PHINode>(I)) return nullptr;
05388 
05389   std::vector<Constant*> Operands(I->getNumOperands());
05390 
05391   for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
05392     Instruction *Operand = dyn_cast<Instruction>(I->getOperand(i));
05393     if (!Operand) {
05394       Operands[i] = dyn_cast<Constant>(I->getOperand(i));
05395       if (!Operands[i]) return nullptr;
05396       continue;
05397     }
05398     Constant *C = EvaluateExpression(Operand, L, Vals, DL, TLI);
05399     Vals[Operand] = C;
05400     if (!C) return nullptr;
05401     Operands[i] = C;
05402   }
05403 
05404   if (CmpInst *CI = dyn_cast<CmpInst>(I))
05405     return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0],
05406                                            Operands[1], DL, TLI);
05407   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
05408     if (!LI->isVolatile())
05409       return ConstantFoldLoadFromConstPtr(Operands[0], DL);
05410   }
05411   return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Operands, DL,
05412                                   TLI);
05413 }
05414 
05415 /// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
05416 /// in the header of its containing loop, we know the loop executes a
05417 /// constant number of times, and the PHI node is just a recurrence
05418 /// involving constants, fold it.
05419 Constant *
05420 ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
05421                                                    const APInt &BEs,
05422                                                    const Loop *L) {
05423   DenseMap<PHINode*, Constant*>::const_iterator I =
05424     ConstantEvolutionLoopExitValue.find(PN);
05425   if (I != ConstantEvolutionLoopExitValue.end())
05426     return I->second;
05427 
05428   if (BEs.ugt(MaxBruteForceIterations))
05429     return ConstantEvolutionLoopExitValue[PN] = nullptr;  // Not going to evaluate it.
05430 
05431   Constant *&RetVal = ConstantEvolutionLoopExitValue[PN];
05432 
05433   DenseMap<Instruction *, Constant *> CurrentIterVals;
05434   BasicBlock *Header = L->getHeader();
05435   assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
05436 
05437   // Since the loop is canonicalized, the PHI node must have two entries.  One
05438   // entry must be a constant (coming in from outside of the loop), and the
05439   // second must be derived from the same PHI.
05440   bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1));
05441   PHINode *PHI = nullptr;
05442   for (BasicBlock::iterator I = Header->begin();
05443        (PHI = dyn_cast<PHINode>(I)); ++I) {
05444     Constant *StartCST =
05445       dyn_cast<Constant>(PHI->getIncomingValue(!SecondIsBackedge));
05446     if (!StartCST) continue;
05447     CurrentIterVals[PHI] = StartCST;
05448   }
05449   if (!CurrentIterVals.count(PN))
05450     return RetVal = nullptr;
05451 
05452   Value *BEValue = PN->getIncomingValue(SecondIsBackedge);
05453 
05454   // Execute the loop symbolically to determine the exit value.
05455   if (BEs.getActiveBits() >= 32)
05456     return RetVal = nullptr; // More than 2^32-1 iterations?? Not doing it!
05457 
05458   unsigned NumIterations = BEs.getZExtValue(); // must be in range
05459   unsigned IterationNum = 0;
05460   for (; ; ++IterationNum) {
05461     if (IterationNum == NumIterations)
05462       return RetVal = CurrentIterVals[PN];  // Got exit value!
05463 
05464     // Compute the value of the PHIs for the next iteration.
05465     // EvaluateExpression adds non-phi values to the CurrentIterVals map.
05466     DenseMap<Instruction *, Constant *> NextIterVals;
05467     Constant *NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL,
05468                                            TLI);
05469     if (!NextPHI)
05470       return nullptr;        // Couldn't evaluate!
05471     NextIterVals[PN] = NextPHI;
05472 
05473     bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
05474 
05475     // Also evaluate the other PHI nodes.  However, we don't get to stop if we
05476     // cease to be able to evaluate one of them or if they stop evolving,
05477     // because that doesn't necessarily prevent us from computing PN.
05478     SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute;
05479     for (DenseMap<Instruction *, Constant *>::const_iterator
05480            I = CurrentIterVals.begin(), E = CurrentIterVals.end(); I != E; ++I){
05481       PHINode *PHI = dyn_cast<PHINode>(I->first);
05482       if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
05483       PHIsToCompute.push_back(std::make_pair(PHI, I->second));
05484     }
05485     // We use two distinct loops because EvaluateExpression may invalidate any
05486     // iterators into CurrentIterVals.
05487     for (SmallVectorImpl<std::pair<PHINode *, Constant*> >::const_iterator
05488              I = PHIsToCompute.begin(), E = PHIsToCompute.end(); I != E; ++I) {
05489       PHINode *PHI = I->first;
05490       Constant *&NextPHI = NextIterVals[PHI];
05491       if (!NextPHI) {   // Not already computed.
05492         Value *BEValue = PHI->getIncomingValue(SecondIsBackedge);
05493         NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, TLI);
05494       }
05495       if (NextPHI != I->second)
05496         StoppedEvolving = false;
05497     }
05498 
05499     // If all entries in CurrentIterVals == NextIterVals then we can stop
05500     // iterating, the loop can't continue to change.
05501     if (StoppedEvolving)
05502       return RetVal = CurrentIterVals[PN];
05503 
05504     CurrentIterVals.swap(NextIterVals);
05505   }
05506 }
05507 
05508 /// ComputeExitCountExhaustively - If the loop is known to execute a
05509 /// constant number of times (the condition evolves only from constants),
05510 /// try to evaluate a few iterations of the loop until we get the exit
05511 /// condition gets a value of ExitWhen (true or false).  If we cannot
05512 /// evaluate the trip count of the loop, return getCouldNotCompute().
05513 const SCEV *ScalarEvolution::ComputeExitCountExhaustively(const Loop *L,
05514                                                           Value *Cond,
05515                                                           bool ExitWhen) {
05516   PHINode *PN = getConstantEvolvingPHI(Cond, L);
05517   if (!PN) return getCouldNotCompute();
05518 
05519   // If the loop is canonicalized, the PHI will have exactly two entries.
05520   // That's the only form we support here.
05521   if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
05522 
05523   DenseMap<Instruction *, Constant *> CurrentIterVals;
05524   BasicBlock *Header = L->getHeader();
05525   assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
05526 
05527   // One entry must be a constant (coming in from outside of the loop), and the
05528   // second must be derived from the same PHI.
05529   bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1));
05530   PHINode *PHI = nullptr;
05531   for (BasicBlock::iterator I = Header->begin();
05532        (PHI = dyn_cast<PHINode>(I)); ++I) {
05533     Constant *StartCST =
05534       dyn_cast<Constant>(PHI->getIncomingValue(!SecondIsBackedge));
05535     if (!StartCST) continue;
05536     CurrentIterVals[PHI] = StartCST;
05537   }
05538   if (!CurrentIterVals.count(PN))
05539     return getCouldNotCompute();
05540 
05541   // Okay, we find a PHI node that defines the trip count of this loop.  Execute
05542   // the loop symbolically to determine when the condition gets a value of
05543   // "ExitWhen".
05544 
05545   unsigned MaxIterations = MaxBruteForceIterations;   // Limit analysis.
05546   for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
05547     ConstantInt *CondVal =
05548       dyn_cast_or_null<ConstantInt>(EvaluateExpression(Cond, L, CurrentIterVals,
05549                                                        DL, TLI));
05550 
05551     // Couldn't symbolically evaluate.
05552     if (!CondVal) return getCouldNotCompute();
05553 
05554     if (CondVal->getValue() == uint64_t(ExitWhen)) {
05555       ++NumBruteForceTripCountsComputed;
05556       return getConstant(Type::getInt32Ty(getContext()), IterationNum);
05557     }
05558 
05559     // Update all the PHI nodes for the next iteration.
05560     DenseMap<Instruction *, Constant *> NextIterVals;
05561 
05562     // Create a list of which PHIs we need to compute. We want to do this before
05563     // calling EvaluateExpression on them because that may invalidate iterators
05564     // into CurrentIterVals.
05565     SmallVector<PHINode *, 8> PHIsToCompute;
05566     for (DenseMap<Instruction *, Constant *>::const_iterator
05567            I = CurrentIterVals.begin(), E = CurrentIterVals.end(); I != E; ++I){
05568       PHINode *PHI = dyn_cast<PHINode>(I->first);
05569       if (!PHI || PHI->getParent() != Header) continue;
05570       PHIsToCompute.push_back(PHI);
05571     }
05572     for (SmallVectorImpl<PHINode *>::const_iterator I = PHIsToCompute.begin(),
05573              E = PHIsToCompute.end(); I != E; ++I) {
05574       PHINode *PHI = *I;
05575       Constant *&NextPHI = NextIterVals[PHI];
05576       if (NextPHI) continue;    // Already computed!
05577 
05578       Value *BEValue = PHI->getIncomingValue(SecondIsBackedge);
05579       NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, TLI);
05580     }
05581     CurrentIterVals.swap(NextIterVals);
05582   }
05583 
05584   // Too many iterations were needed to evaluate.
05585   return getCouldNotCompute();
05586 }
05587 
05588 /// getSCEVAtScope - Return a SCEV expression for the specified value
05589 /// at the specified scope in the program.  The L value specifies a loop
05590 /// nest to evaluate the expression at, where null is the top-level or a
05591 /// specified loop is immediately inside of the loop.
05592 ///
05593 /// This method can be used to compute the exit value for a variable defined
05594 /// in a loop by querying what the value will hold in the parent loop.
05595 ///
05596 /// In the case that a relevant loop exit value cannot be computed, the
05597 /// original value V is returned.
05598 const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
05599   // Check to see if we've folded this expression at this loop before.
05600   SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values = ValuesAtScopes[V];
05601   for (unsigned u = 0; u < Values.size(); u++) {
05602     if (Values[u].first == L)
05603       return Values[u].second ? Values[u].second : V;
05604   }
05605   Values.push_back(std::make_pair(L, static_cast<const SCEV *>(nullptr)));
05606   // Otherwise compute it.
05607   const SCEV *C = computeSCEVAtScope(V, L);
05608   SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values2 = ValuesAtScopes[V];
05609   for (unsigned u = Values2.size(); u > 0; u--) {
05610     if (Values2[u - 1].first == L) {
05611       Values2[u - 1].second = C;
05612       break;
05613     }
05614   }
05615   return C;
05616 }
05617 
05618 /// This builds up a Constant using the ConstantExpr interface.  That way, we
05619 /// will return Constants for objects which aren't represented by a
05620 /// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
05621 /// Returns NULL if the SCEV isn't representable as a Constant.
05622 static Constant *BuildConstantFromSCEV(const SCEV *V) {
05623   switch (static_cast<SCEVTypes>(V->getSCEVType())) {
05624     case scCouldNotCompute:
05625     case scAddRecExpr:
05626       break;
05627     case scConstant:
05628       return cast<SCEVConstant>(V)->getValue();
05629     case scUnknown:
05630       return dyn_cast<Constant>(cast<SCEVUnknown>(V)->getValue());
05631     case scSignExtend: {
05632       const SCEVSignExtendExpr *SS = cast<SCEVSignExtendExpr>(V);
05633       if (Constant *CastOp = BuildConstantFromSCEV(SS->getOperand()))
05634         return ConstantExpr::getSExt(CastOp, SS->getType());
05635       break;
05636     }
05637     case scZeroExtend: {
05638       const SCEVZeroExtendExpr *SZ = cast<SCEVZeroExtendExpr>(V);
05639       if (Constant *CastOp = BuildConstantFromSCEV(SZ->getOperand()))
05640         return ConstantExpr::getZExt(CastOp, SZ->getType());
05641       break;
05642     }
05643     case scTruncate: {
05644       const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(V);
05645       if (Constant *CastOp = BuildConstantFromSCEV(ST->getOperand()))
05646         return ConstantExpr::getTrunc(CastOp, ST->getType());
05647       break;
05648     }
05649     case scAddExpr: {
05650       const SCEVAddExpr *SA = cast<SCEVAddExpr>(V);
05651       if (Constant *C = BuildConstantFromSCEV(SA->getOperand(0))) {
05652         if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) {
05653           unsigned AS = PTy->getAddressSpace();
05654           Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS);
05655           C = ConstantExpr::getBitCast(C, DestPtrTy);
05656         }
05657         for (unsigned i = 1, e = SA->getNumOperands(); i != e; ++i) {
05658           Constant *C2 = BuildConstantFromSCEV(SA->getOperand(i));
05659           if (!C2) return nullptr;
05660 
05661           // First pointer!
05662           if (!C->getType()->isPointerTy() && C2->getType()->isPointerTy()) {
05663             unsigned AS = C2->getType()->getPointerAddressSpace();
05664             std::swap(C, C2);
05665             Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS);
05666             // The offsets have been converted to bytes.  We can add bytes to an
05667             // i8* by GEP with the byte count in the first index.
05668             C = ConstantExpr::getBitCast(C, DestPtrTy);
05669           }
05670 
05671           // Don't bother trying to sum two pointers. We probably can't
05672           // statically compute a load that results from it anyway.
05673           if (C2->getType()->isPointerTy())
05674             return nullptr;
05675 
05676           if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) {
05677             if (PTy->getElementType()->isStructTy())
05678               C2 = ConstantExpr::getIntegerCast(
05679                   C2, Type::getInt32Ty(C->getContext()), true);
05680             C = ConstantExpr::getGetElementPtr(C, C2);
05681           } else
05682             C = ConstantExpr::getAdd(C, C2);
05683         }
05684         return C;
05685       }
05686       break;
05687     }
05688     case scMulExpr: {
05689       const SCEVMulExpr *SM = cast<SCEVMulExpr>(V);
05690       if (Constant *C = BuildConstantFromSCEV(SM->getOperand(0))) {
05691         // Don't bother with pointers at all.
05692         if (C->getType()->isPointerTy()) return nullptr;
05693         for (unsigned i = 1, e = SM->getNumOperands(); i != e; ++i) {
05694           Constant *C2 = BuildConstantFromSCEV(SM->getOperand(i));
05695           if (!C2 || C2->getType()->isPointerTy()) return nullptr;
05696           C = ConstantExpr::getMul(C, C2);
05697         }
05698         return C;
05699       }
05700       break;
05701     }
05702     case scUDivExpr: {
05703       const SCEVUDivExpr *SU = cast<SCEVUDivExpr>(V);
05704       if (Constant *LHS = BuildConstantFromSCEV(SU->getLHS()))
05705         if (Constant *RHS = BuildConstantFromSCEV(SU->getRHS()))
05706           if (LHS->getType() == RHS->getType())
05707             return ConstantExpr::getUDiv(LHS, RHS);
05708       break;
05709     }
05710     case scSMaxExpr:
05711     case scUMaxExpr:
05712       break; // TODO: smax, umax.
05713   }
05714   return nullptr;
05715 }
05716 
05717 const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
05718   if (isa<SCEVConstant>(V)) return V;
05719 
05720   // If this instruction is evolved from a constant-evolving PHI, compute the
05721   // exit value from the loop without using SCEVs.
05722   if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) {
05723     if (Instruction *I = dyn_cast<Instruction>(SU->getValue())) {
05724       const Loop *LI = (*this->LI)[I->getParent()];
05725       if (LI && LI->getParentLoop() == L)  // Looking for loop exit value.
05726         if (PHINode *PN = dyn_cast<PHINode>(I))
05727           if (PN->getParent() == LI->getHeader()) {
05728             // Okay, there is no closed form solution for the PHI node.  Check
05729             // to see if the loop that contains it has a known backedge-taken
05730             // count.  If so, we may be able to force computation of the exit
05731             // value.
05732             const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI);
05733             if (const SCEVConstant *BTCC =
05734                   dyn_cast<SCEVConstant>(BackedgeTakenCount)) {
05735               // Okay, we know how many times the containing loop executes.  If
05736               // this is a constant evolving PHI node, get the final value at
05737               // the specified iteration number.
05738               Constant *RV = getConstantEvolutionLoopExitValue(PN,
05739                                                    BTCC->getValue()->getValue(),
05740                                                                LI);
05741               if (RV) return getSCEV(RV);
05742             }
05743           }
05744 
05745       // Okay, this is an expression that we cannot symbolically evaluate
05746       // into a SCEV.  Check to see if it's possible to symbolically evaluate
05747       // the arguments into constants, and if so, try to constant propagate the
05748       // result.  This is particularly useful for computing loop exit values.
05749       if (CanConstantFold(I)) {
05750         SmallVector<Constant *, 4> Operands;
05751         bool MadeImprovement = false;
05752         for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
05753           Value *Op = I->getOperand(i);
05754           if (Constant *C = dyn_cast<Constant>(Op)) {
05755             Operands.push_back(C);
05756             continue;
05757           }
05758 
05759           // If any of the operands is non-constant and if they are
05760           // non-integer and non-pointer, don't even try to analyze them
05761           // with scev techniques.
05762           if (!isSCEVable(Op->getType()))
05763             return V;
05764 
05765           const SCEV *OrigV = getSCEV(Op);
05766           const SCEV *OpV = getSCEVAtScope(OrigV, L);
05767           MadeImprovement |= OrigV != OpV;
05768 
05769           Constant *C = BuildConstantFromSCEV(OpV);
05770           if (!C) return V;
05771           if (C->getType() != Op->getType())
05772             C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false,
05773                                                               Op->getType(),
05774                                                               false),
05775                                       C, Op->getType());
05776           Operands.push_back(C);
05777         }
05778 
05779         // Check to see if getSCEVAtScope actually made an improvement.
05780         if (MadeImprovement) {
05781           Constant *C = nullptr;
05782           if (const CmpInst *CI = dyn_cast<CmpInst>(I))
05783             C = ConstantFoldCompareInstOperands(CI->getPredicate(),
05784                                                 Operands[0], Operands[1], DL,
05785                                                 TLI);
05786           else if (const LoadInst *LI = dyn_cast<LoadInst>(I)) {
05787             if (!LI->isVolatile())
05788               C = ConstantFoldLoadFromConstPtr(Operands[0], DL);
05789           } else
05790             C = ConstantFoldInstOperands(I->getOpcode(), I->getType(),
05791                                          Operands, DL, TLI);
05792           if (!C) return V;
05793           return getSCEV(C);
05794         }
05795       }
05796     }
05797 
05798     // This is some other type of SCEVUnknown, just return it.
05799     return V;
05800   }
05801 
05802   if (const SCEVCommutativeExpr *Comm = dyn_cast<SCEVCommutativeExpr>(V)) {
05803     // Avoid performing the look-up in the common case where the specified
05804     // expression has no loop-variant portions.
05805     for (unsigned i = 0, e = Comm->getNumOperands(); i != e; ++i) {
05806       const SCEV *OpAtScope = getSCEVAtScope(Comm->getOperand(i), L);
05807       if (OpAtScope != Comm->getOperand(i)) {
05808         // Okay, at least one of these operands is loop variant but might be
05809         // foldable.  Build a new instance of the folded commutative expression.
05810         SmallVector<const SCEV *, 8> NewOps(Comm->op_begin(),
05811                                             Comm->op_begin()+i);
05812         NewOps.push_back(OpAtScope);
05813 
05814         for (++i; i != e; ++i) {
05815           OpAtScope = getSCEVAtScope(Comm->getOperand(i), L);
05816           NewOps.push_back(OpAtScope);
05817         }
05818         if (isa<SCEVAddExpr>(Comm))
05819           return getAddExpr(NewOps);
05820         if (isa<SCEVMulExpr>(Comm))
05821           return getMulExpr(NewOps);
05822         if (isa<SCEVSMaxExpr>(Comm))
05823           return getSMaxExpr(NewOps);
05824         if (isa<SCEVUMaxExpr>(Comm))
05825           return getUMaxExpr(NewOps);
05826         llvm_unreachable("Unknown commutative SCEV type!");
05827       }
05828     }
05829     // If we got here, all operands are loop invariant.
05830     return Comm;
05831   }
05832 
05833   if (const SCEVUDivExpr *Div = dyn_cast<SCEVUDivExpr>(V)) {
05834     const SCEV *LHS = getSCEVAtScope(Div->getLHS(), L);
05835     const SCEV *RHS = getSCEVAtScope(Div->getRHS(), L);
05836     if (LHS == Div->getLHS() && RHS == Div->getRHS())
05837       return Div;   // must be loop invariant
05838     return getUDivExpr(LHS, RHS);
05839   }
05840 
05841   // If this is a loop recurrence for a loop that does not contain L, then we
05842   // are dealing with the final value computed by the loop.
05843   if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V)) {
05844     // First, attempt to evaluate each operand.
05845     // Avoid performing the look-up in the common case where the specified
05846     // expression has no loop-variant portions.
05847     for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
05848       const SCEV *OpAtScope = getSCEVAtScope(AddRec->getOperand(i), L);
05849       if (OpAtScope == AddRec->getOperand(i))
05850         continue;
05851 
05852       // Okay, at least one of these operands is loop variant but might be
05853       // foldable.  Build a new instance of the folded commutative expression.
05854       SmallVector<const SCEV *, 8> NewOps(AddRec->op_begin(),
05855                                           AddRec->op_begin()+i);
05856       NewOps.push_back(OpAtScope);
05857       for (++i; i != e; ++i)
05858         NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L));
05859 
05860       const SCEV *FoldedRec =
05861         getAddRecExpr(NewOps, AddRec->getLoop(),
05862                       AddRec->getNoWrapFlags(SCEV::FlagNW));
05863       AddRec = dyn_cast<SCEVAddRecExpr>(FoldedRec);
05864       // The addrec may be folded to a nonrecurrence, for example, if the
05865       // induction variable is multiplied by zero after constant folding. Go
05866       // ahead and return the folded value.
05867       if (!AddRec)
05868         return FoldedRec;
05869       break;
05870     }
05871 
05872     // If the scope is outside the addrec's loop, evaluate it by using the
05873     // loop exit value of the addrec.
05874     if (!AddRec->getLoop()->contains(L)) {
05875       // To evaluate this recurrence, we need to know how many times the AddRec
05876       // loop iterates.  Compute this now.
05877       const SCEV *BackedgeTakenCount = getBackedgeTakenCount(AddRec->getLoop());
05878       if (BackedgeTakenCount == getCouldNotCompute()) return AddRec;
05879 
05880       // Then, evaluate the AddRec.
05881       return AddRec->evaluateAtIteration(BackedgeTakenCount, *this);
05882     }
05883 
05884     return AddRec;
05885   }
05886 
05887   if (const SCEVZeroExtendExpr *Cast = dyn_cast<SCEVZeroExtendExpr>(V)) {
05888     const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
05889     if (Op == Cast->getOperand())
05890       return Cast;  // must be loop invariant
05891     return getZeroExtendExpr(Op, Cast->getType());
05892   }
05893 
05894   if (const SCEVSignExtendExpr *Cast = dyn_cast<SCEVSignExtendExpr>(V)) {
05895     const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
05896     if (Op == Cast->getOperand())
05897       return Cast;  // must be loop invariant
05898     return getSignExtendExpr(Op, Cast->getType());
05899   }
05900 
05901   if (const SCEVTruncateExpr *Cast = dyn_cast<SCEVTruncateExpr>(V)) {
05902     const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
05903     if (Op == Cast->getOperand())
05904       return Cast;  // must be loop invariant
05905     return getTruncateExpr(Op, Cast->getType());
05906   }
05907 
05908   llvm_unreachable("Unknown SCEV type!");
05909 }
05910 
05911 /// getSCEVAtScope - This is a convenience function which does
05912 /// getSCEVAtScope(getSCEV(V), L).
05913 const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) {
05914   return getSCEVAtScope(getSCEV(V), L);
05915 }
05916 
05917 /// SolveLinEquationWithOverflow - Finds the minimum unsigned root of the
05918 /// following equation:
05919 ///
05920 ///     A * X = B (mod N)
05921 ///
05922 /// where N = 2^BW and BW is the common bit width of A and B. The signedness of
05923 /// A and B isn't important.
05924 ///
05925 /// If the equation does not have a solution, SCEVCouldNotCompute is returned.
05926 static const SCEV *SolveLinEquationWithOverflow(const APInt &A, const APInt &B,
05927                                                ScalarEvolution &SE) {
05928   uint32_t BW = A.getBitWidth();
05929   assert(BW == B.getBitWidth() && "Bit widths must be the same.");
05930   assert(A != 0 && "A must be non-zero.");
05931 
05932   // 1. D = gcd(A, N)
05933   //
05934   // The gcd of A and N may have only one prime factor: 2. The number of
05935   // trailing zeros in A is its multiplicity
05936   uint32_t Mult2 = A.countTrailingZeros();
05937   // D = 2^Mult2
05938 
05939   // 2. Check if B is divisible by D.
05940   //
05941   // B is divisible by D if and only if the multiplicity of prime factor 2 for B
05942   // is not less than multiplicity of this prime factor for D.
05943   if (B.countTrailingZeros() < Mult2)
05944     return SE.getCouldNotCompute();
05945 
05946   // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
05947   // modulo (N / D).
05948   //
05949   // (N / D) may need BW+1 bits in its representation.  Hence, we'll use this
05950   // bit width during computations.
05951   APInt AD = A.lshr(Mult2).zext(BW + 1);  // AD = A / D
05952   APInt Mod(BW + 1, 0);
05953   Mod.setBit(BW - Mult2);  // Mod = N / D
05954   APInt I = AD.multiplicativeInverse(Mod);
05955 
05956   // 4. Compute the minimum unsigned root of the equation:
05957   // I * (B / D) mod (N / D)
05958   APInt Result = (I * B.lshr(Mult2).zext(BW + 1)).urem(Mod);
05959 
05960   // The result is guaranteed to be less than 2^BW so we may truncate it to BW
05961   // bits.
05962   return SE.getConstant(Result.trunc(BW));
05963 }
05964 
05965 /// SolveQuadraticEquation - Find the roots of the quadratic equation for the
05966 /// given quadratic chrec {L,+,M,+,N}.  This returns either the two roots (which
05967 /// might be the same) or two SCEVCouldNotCompute objects.
05968 ///
05969 static std::pair<const SCEV *,const SCEV *>
05970 SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
05971   assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
05972   const SCEVConstant *LC = dyn_cast<SCEVConstant>(AddRec->getOperand(0));
05973   const SCEVConstant *MC = dyn_cast<SCEVConstant>(AddRec->getOperand(1));
05974   const SCEVConstant *NC = dyn_cast<SCEVConstant>(AddRec->getOperand(2));
05975 
05976   // We currently can only solve this if the coefficients are constants.
05977   if (!LC || !MC || !NC) {
05978     const SCEV *CNC = SE.getCouldNotCompute();
05979     return std::make_pair(CNC, CNC);
05980   }
05981 
05982   uint32_t BitWidth = LC->getValue()->getValue().getBitWidth();
05983   const APInt &L = LC->getValue()->getValue();
05984   const APInt &M = MC->getValue()->getValue();
05985   const APInt &N = NC->getValue()->getValue();
05986   APInt Two(BitWidth, 2);
05987   APInt Four(BitWidth, 4);
05988 
05989   {
05990     using namespace APIntOps;
05991     const APInt& C = L;
05992     // Convert from chrec coefficients to polynomial coefficients AX^2+BX+C
05993     // The B coefficient is M-N/2
05994     APInt B(M);
05995     B -= sdiv(N,Two);
05996 
05997     // The A coefficient is N/2
05998     APInt A(N.sdiv(Two));
05999 
06000     // Compute the B^2-4ac term.
06001     APInt SqrtTerm(B);
06002     SqrtTerm *= B;
06003     SqrtTerm -= Four * (A * C);
06004 
06005     if (SqrtTerm.isNegative()) {
06006       // The loop is provably infinite.
06007       const SCEV *CNC = SE.getCouldNotCompute();
06008       return std::make_pair(CNC, CNC);
06009     }
06010 
06011     // Compute sqrt(B^2-4ac). This is guaranteed to be the nearest
06012     // integer value or else APInt::sqrt() will assert.
06013     APInt SqrtVal(SqrtTerm.sqrt());
06014 
06015     // Compute the two solutions for the quadratic formula.
06016     // The divisions must be performed as signed divisions.
06017     APInt NegB(-B);
06018     APInt TwoA(A << 1);
06019     if (TwoA.isMinValue()) {
06020       const SCEV *CNC = SE.getCouldNotCompute();
06021       return std::make_pair(CNC, CNC);
06022     }
06023 
06024     LLVMContext &Context = SE.getContext();
06025 
06026     ConstantInt *Solution1 =
06027       ConstantInt::get(Context, (NegB + SqrtVal).sdiv(TwoA));
06028     ConstantInt *Solution2 =
06029       ConstantInt::get(Context, (NegB - SqrtVal).sdiv(TwoA));
06030 
06031     return std::make_pair(SE.getConstant(Solution1),
06032                           SE.getConstant(Solution2));
06033   } // end APIntOps namespace
06034 }
06035 
06036 /// HowFarToZero - Return the number of times a backedge comparing the specified
06037 /// value to zero will execute.  If not computable, return CouldNotCompute.
06038 ///
06039 /// This is only used for loops with a "x != y" exit test. The exit condition is
06040 /// now expressed as a single expression, V = x-y. So the exit test is
06041 /// effectively V != 0.  We know and take advantage of the fact that this
06042 /// expression only being used in a comparison by zero context.
06043 ScalarEvolution::ExitLimit
06044 ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool ControlsExit) {
06045   // If the value is a constant
06046   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
06047     // If the value is already zero, the branch will execute zero times.
06048     if (C->getValue()->isZero()) return C;
06049     return getCouldNotCompute();  // Otherwise it will loop infinitely.
06050   }
06051 
06052   const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V);
06053   if (!AddRec || AddRec->getLoop() != L)
06054     return getCouldNotCompute();
06055 
06056   // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
06057   // the quadratic equation to solve it.
06058   if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
06059     std::pair<const SCEV *,const SCEV *> Roots =
06060       SolveQuadraticEquation(AddRec, *this);
06061     const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first);
06062     const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second);
06063     if (R1 && R2) {
06064 #if 0
06065       dbgs() << "HFTZ: " << *V << " - sol#1: " << *R1
06066              << "  sol#2: " << *R2 << "\n";
06067 #endif
06068       // Pick the smallest positive root value.
06069       if (ConstantInt *CB =
06070           dyn_cast<ConstantInt>(ConstantExpr::getICmp(CmpInst::ICMP_ULT,
06071                                                       R1->getValue(),
06072                                                       R2->getValue()))) {
06073         if (CB->getZExtValue() == false)
06074           std::swap(R1, R2);   // R1 is the minimum root now.
06075 
06076         // We can only use this value if the chrec ends up with an exact zero
06077         // value at this index.  When solving for "X*X != 5", for example, we
06078         // should not accept a root of 2.
06079         const SCEV *Val = AddRec->evaluateAtIteration(R1, *this);
06080         if (Val->isZero())
06081           return R1;  // We found a quadratic root!
06082       }
06083     }
06084     return getCouldNotCompute();
06085   }
06086 
06087   // Otherwise we can only handle this if it is affine.
06088   if (!AddRec->isAffine())
06089     return getCouldNotCompute();
06090 
06091   // If this is an affine expression, the execution count of this branch is
06092   // the minimum unsigned root of the following equation:
06093   //
06094   //     Start + Step*N = 0 (mod 2^BW)
06095   //
06096   // equivalent to:
06097   //
06098   //             Step*N = -Start (mod 2^BW)
06099   //
06100   // where BW is the common bit width of Start and Step.
06101 
06102   // Get the initial value for the loop.
06103   const SCEV *Start = getSCEVAtScope(AddRec->getStart(), L->getParentLoop());
06104   const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), L->getParentLoop());
06105 
06106   // For now we handle only constant steps.
06107   //
06108   // TODO: Handle a nonconstant Step given AddRec<NUW>. If the
06109   // AddRec is NUW, then (in an unsigned sense) it cannot be counting up to wrap
06110   // to 0, it must be counting down to equal 0. Consequently, N = Start / -Step.
06111   // We have not yet seen any such cases.
06112   const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Step);
06113   if (!StepC || StepC->getValue()->equalsInt(0))
06114     return getCouldNotCompute();
06115 
06116   // For positive steps (counting up until unsigned overflow):
06117   //   N = -Start/Step (as unsigned)
06118   // For negative steps (counting down to zero):
06119   //   N = Start/-Step
06120   // First compute the unsigned distance from zero in the direction of Step.
06121   bool CountDown = StepC->getValue()->getValue().isNegative();
06122   const SCEV *Distance = CountDown ? Start : getNegativeSCEV(Start);
06123 
06124   // Handle unitary steps, which cannot wraparound.
06125   // 1*N = -Start; -1*N = Start (mod 2^BW), so:
06126   //   N = Distance (as unsigned)
06127   if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue()) {
06128     ConstantRange CR = getUnsignedRange(Start);
06129     const SCEV *MaxBECount;
06130     if (!CountDown && CR.getUnsignedMin().isMinValue())
06131       // When counting up, the worst starting value is 1, not 0.
06132       MaxBECount = CR.getUnsignedMax().isMinValue()
06133         ? getConstant(APInt::getMinValue(CR.getBitWidth()))
06134         : getConstant(APInt::getMaxValue(CR.getBitWidth()));
06135     else
06136       MaxBECount = getConstant(CountDown ? CR.getUnsignedMax()
06137                                          : -CR.getUnsignedMin());
06138     return ExitLimit(Distance, MaxBECount);
06139   }
06140 
06141   // If the step exactly divides the distance then unsigned divide computes the
06142   // backedge count.
06143   const SCEV *Q, *R;
06144   ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
06145   SCEVUDivision::divide(SE, Distance, Step, &Q, &R);
06146   if (R->isZero()) {
06147     const SCEV *Exact =
06148         getUDivExactExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step);
06149     return ExitLimit(Exact, Exact);
06150   }
06151 
06152   // If the condition controls loop exit (the loop exits only if the expression
06153   // is true) and the addition is no-wrap we can use unsigned divide to
06154   // compute the backedge count.  In this case, the step may not divide the
06155   // distance, but we don't care because if the condition is "missed" the loop
06156   // will have undefined behavior due to wrapping.
06157   if (ControlsExit && AddRec->getNoWrapFlags(SCEV::FlagNW)) {
06158     const SCEV *Exact =
06159         getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step);
06160     return ExitLimit(Exact, Exact);
06161   }
06162 
06163   // Then, try to solve the above equation provided that Start is constant.
06164   if (const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start))
06165     return SolveLinEquationWithOverflow(StepC->getValue()->getValue(),
06166                                         -StartC->getValue()->getValue(),
06167                                         *this);
06168   return getCouldNotCompute();
06169 }
06170 
06171 /// HowFarToNonZero - Return the number of times a backedge checking the
06172 /// specified value for nonzero will execute.  If not computable, return
06173 /// CouldNotCompute
06174 ScalarEvolution::ExitLimit
06175 ScalarEvolution::HowFarToNonZero(const SCEV *V, const Loop *L) {
06176   // Loops that look like: while (X == 0) are very strange indeed.  We don't
06177   // handle them yet except for the trivial case.  This could be expanded in the
06178   // future as needed.
06179 
06180   // If the value is a constant, check to see if it is known to be non-zero
06181   // already.  If so, the backedge will execute zero times.
06182   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
06183     if (!C->getValue()->isNullValue())
06184       return getConstant(C->getType(), 0);
06185     return getCouldNotCompute();  // Otherwise it will loop infinitely.
06186   }
06187 
06188   // We could implement others, but I really doubt anyone writes loops like
06189   // this, and if they did, they would already be constant folded.
06190   return getCouldNotCompute();
06191 }
06192 
06193 /// getPredecessorWithUniqueSuccessorForBB - Return a predecessor of BB
06194 /// (which may not be an immediate predecessor) which has exactly one
06195 /// successor from which BB is reachable, or null if no such block is
06196 /// found.
06197 ///
06198 std::pair<BasicBlock *, BasicBlock *>
06199 ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(BasicBlock *BB) {
06200   // If the block has a unique predecessor, then there is no path from the
06201   // predecessor to the block that does not go through the direct edge
06202   // from the predecessor to the block.
06203   if (BasicBlock *Pred = BB->getSinglePredecessor())
06204     return std::make_pair(Pred, BB);
06205 
06206   // A loop's header is defined to be a block that dominates the loop.
06207   // If the header has a unique predecessor outside the loop, it must be
06208   // a block that has exactly one successor that can reach the loop.
06209   if (Loop *L = LI->getLoopFor(BB))
06210     return std::make_pair(L->getLoopPredecessor(), L->getHeader());
06211 
06212   return std::pair<BasicBlock *, BasicBlock *>();
06213 }
06214 
06215 /// HasSameValue - SCEV structural equivalence is usually sufficient for
06216 /// testing whether two expressions are equal, however for the purposes of
06217 /// looking for a condition guarding a loop, it can be useful to be a little
06218 /// more general, since a front-end may have replicated the controlling
06219 /// expression.
06220 ///
06221 static bool HasSameValue(const SCEV *A, const SCEV *B) {
06222   // Quick check to see if they are the same SCEV.
06223   if (A == B) return true;
06224 
06225   // Otherwise, if they're both SCEVUnknown, it's possible that they hold
06226   // two different instructions with the same value. Check for this case.
06227   if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A))
06228     if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B))
06229       if (const Instruction *AI = dyn_cast<Instruction>(AU->getValue()))
06230         if (const Instruction *BI = dyn_cast<Instruction>(BU->getValue()))
06231           if (AI->isIdenticalTo(BI) && !AI->mayReadFromMemory())
06232             return true;
06233 
06234   // Otherwise assume they may have a different value.
06235   return false;
06236 }
06237 
06238 /// SimplifyICmpOperands - Simplify LHS and RHS in a comparison with
06239 /// predicate Pred. Return true iff any changes were made.
06240 ///
06241 bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred,
06242                                            const SCEV *&LHS, const SCEV *&RHS,
06243                                            unsigned Depth) {
06244   bool Changed = false;
06245 
06246   // If we hit the max recursion limit bail out.
06247   if (Depth >= 3)
06248     return false;
06249 
06250   // Canonicalize a constant to the right side.
06251   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
06252     // Check for both operands constant.
06253     if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
06254       if (ConstantExpr::getICmp(Pred,
06255                                 LHSC->getValue(),
06256                                 RHSC->getValue())->isNullValue())
06257         goto trivially_false;
06258       else
06259         goto trivially_true;
06260     }
06261     // Otherwise swap the operands to put the constant on the right.
06262     std::swap(LHS, RHS);
06263     Pred = ICmpInst::getSwappedPredicate(Pred);
06264     Changed = true;
06265   }
06266 
06267   // If we're comparing an addrec with a value which is loop-invariant in the
06268   // addrec's loop, put the addrec on the left. Also make a dominance check,
06269   // as both operands could be addrecs loop-invariant in each other's loop.
06270   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) {
06271     const Loop *L = AR->getLoop();
06272     if (isLoopInvariant(LHS, L) && properlyDominates(LHS, L->getHeader())) {
06273       std::swap(LHS, RHS);
06274       Pred = ICmpInst::getSwappedPredicate(Pred);
06275       Changed = true;
06276     }
06277   }
06278 
06279   // If there's a constant operand, canonicalize comparisons with boundary
06280   // cases, and canonicalize *-or-equal comparisons to regular comparisons.
06281   if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) {
06282     const APInt &RA = RC->getValue()->getValue();
06283     switch (Pred) {
06284     default: llvm_unreachable("Unexpected ICmpInst::Predicate value!");
06285     case ICmpInst::ICMP_EQ:
06286     case ICmpInst::ICMP_NE:
06287       // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
06288       if (!RA)
06289         if (const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(LHS))
06290           if (const SCEVMulExpr *ME = dyn_cast<SCEVMulExpr>(AE->getOperand(0)))
06291             if (AE->getNumOperands() == 2 && ME->getNumOperands() == 2 &&
06292                 ME->getOperand(0)->isAllOnesValue()) {
06293               RHS = AE->getOperand(1);
06294               LHS = ME->getOperand(1);
06295               Changed = true;
06296             }
06297       break;
06298     case ICmpInst::ICMP_UGE:
06299       if ((RA - 1).isMinValue()) {
06300         Pred = ICmpInst::ICMP_NE;
06301         RHS = getConstant(RA - 1);
06302         Changed = true;
06303         break;
06304       }
06305       if (RA.isMaxValue()) {
06306         Pred = ICmpInst::ICMP_EQ;
06307         Changed = true;
06308         break;
06309       }
06310       if (RA.isMinValue()) goto trivially_true;
06311 
06312       Pred = ICmpInst::ICMP_UGT;
06313       RHS = getConstant(RA - 1);
06314       Changed = true;
06315       break;
06316     case ICmpInst::ICMP_ULE:
06317       if ((RA + 1).isMaxValue()) {
06318         Pred = ICmpInst::ICMP_NE;
06319         RHS = getConstant(RA + 1);
06320         Changed = true;
06321         break;
06322       }
06323       if (RA.isMinValue()) {
06324         Pred = ICmpInst::ICMP_EQ;
06325         Changed = true;
06326         break;
06327       }
06328       if (RA.isMaxValue()) goto trivially_true;
06329 
06330       Pred = ICmpInst::ICMP_ULT;
06331       RHS = getConstant(RA + 1);
06332       Changed = true;
06333       break;
06334     case ICmpInst::ICMP_SGE:
06335       if ((RA - 1).isMinSignedValue()) {
06336         Pred = ICmpInst::ICMP_NE;
06337         RHS = getConstant(RA - 1);
06338         Changed = true;
06339         break;
06340       }
06341       if (RA.isMaxSignedValue()) {
06342         Pred = ICmpInst::ICMP_EQ;
06343         Changed = true;
06344         break;
06345       }
06346       if (RA.isMinSignedValue()) goto trivially_true;
06347 
06348       Pred = ICmpInst::ICMP_SGT;
06349       RHS = getConstant(RA - 1);
06350       Changed = true;
06351       break;
06352     case ICmpInst::ICMP_SLE:
06353       if ((RA + 1).isMaxSignedValue()) {
06354         Pred = ICmpInst::ICMP_NE;
06355         RHS = getConstant(RA + 1);
06356         Changed = true;
06357         break;
06358       }
06359       if (RA.isMinSignedValue()) {
06360         Pred = ICmpInst::ICMP_EQ;
06361         Changed = true;
06362         break;
06363       }
06364       if (RA.isMaxSignedValue()) goto trivially_true;
06365 
06366       Pred = ICmpInst::ICMP_SLT;
06367       RHS = getConstant(RA + 1);
06368       Changed = true;
06369       break;
06370     case ICmpInst::ICMP_UGT:
06371       if (RA.isMinValue()) {
06372         Pred = ICmpInst::ICMP_NE;
06373         Changed = true;
06374         break;
06375       }
06376       if ((RA + 1).isMaxValue()) {
06377         Pred = ICmpInst::ICMP_EQ;
06378         RHS = getConstant(RA + 1);
06379         Changed = true;
06380         break;
06381       }
06382       if (RA.isMaxValue()) goto trivially_false;
06383       break;
06384     case ICmpInst::ICMP_ULT:
06385       if (RA.isMaxValue()) {
06386         Pred = ICmpInst::ICMP_NE;
06387         Changed = true;
06388         break;
06389       }
06390       if ((RA - 1).isMinValue()) {
06391         Pred = ICmpInst::ICMP_EQ;
06392         RHS = getConstant(RA - 1);
06393         Changed = true;
06394         break;
06395       }
06396       if (RA.isMinValue()) goto trivially_false;
06397       break;
06398     case ICmpInst::ICMP_SGT:
06399       if (RA.isMinSignedValue()) {
06400         Pred = ICmpInst::ICMP_NE;
06401         Changed = true;
06402         break;
06403       }
06404       if ((RA + 1).isMaxSignedValue()) {
06405         Pred = ICmpInst::ICMP_EQ;
06406         RHS = getConstant(RA + 1);
06407         Changed = true;
06408         break;
06409       }
06410       if (RA.isMaxSignedValue()) goto trivially_false;
06411       break;
06412     case ICmpInst::ICMP_SLT:
06413       if (RA.isMaxSignedValue()) {
06414         Pred = ICmpInst::ICMP_NE;
06415         Changed = true;
06416         break;
06417       }
06418       if ((RA - 1).isMinSignedValue()) {
06419        Pred = ICmpInst::ICMP_EQ;
06420        RHS = getConstant(RA - 1);
06421         Changed = true;
06422        break;
06423       }
06424       if (RA.isMinSignedValue()) goto trivially_false;
06425       break;
06426     }
06427   }
06428 
06429   // Check for obvious equality.
06430   if (HasSameValue(LHS, RHS)) {
06431     if (ICmpInst::isTrueWhenEqual(Pred))
06432       goto trivially_true;
06433     if (ICmpInst::isFalseWhenEqual(Pred))
06434       goto trivially_false;
06435   }
06436 
06437   // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
06438   // adding or subtracting 1 from one of the operands.
06439   switch (Pred) {
06440   case ICmpInst::ICMP_SLE:
06441     if (!getSignedRange(RHS).getSignedMax().isMaxSignedValue()) {
06442       RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
06443                        SCEV::FlagNSW);
06444       Pred = ICmpInst::ICMP_SLT;
06445       Changed = true;
06446     } else if (!getSignedRange(LHS).getSignedMin().isMinSignedValue()) {
06447       LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS,
06448                        SCEV::FlagNSW);
06449       Pred = ICmpInst::ICMP_SLT;
06450       Changed = true;
06451     }
06452     break;
06453   case ICmpInst::ICMP_SGE:
06454     if (!getSignedRange(RHS).getSignedMin().isMinSignedValue()) {
06455       RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS,
06456                        SCEV::FlagNSW);
06457       Pred = ICmpInst::ICMP_SGT;
06458       Changed = true;
06459     } else if (!getSignedRange(LHS).getSignedMax().isMaxSignedValue()) {
06460       LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
06461                        SCEV::FlagNSW);
06462       Pred = ICmpInst::ICMP_SGT;
06463       Changed = true;
06464     }
06465     break;
06466   case ICmpInst::ICMP_ULE:
06467     if (!getUnsignedRange(RHS).getUnsignedMax().isMaxValue()) {
06468       RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
06469                        SCEV::FlagNUW);
06470       Pred = ICmpInst::ICMP_ULT;
06471       Changed = true;
06472     } else if (!getUnsignedRange(LHS).getUnsignedMin().isMinValue()) {
06473       LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS,
06474                        SCEV::FlagNUW);
06475       Pred = ICmpInst::ICMP_ULT;
06476       Changed = true;
06477     }
06478     break;
06479   case ICmpInst::ICMP_UGE:
06480     if (!getUnsignedRange(RHS).getUnsignedMin().isMinValue()) {
06481       RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS,
06482                        SCEV::FlagNUW);
06483       Pred = ICmpInst::ICMP_UGT;
06484       Changed = true;
06485     } else if (!getUnsignedRange(LHS).getUnsignedMax().isMaxValue()) {
06486       LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
06487                        SCEV::FlagNUW);
06488       Pred = ICmpInst::ICMP_UGT;
06489       Changed = true;
06490     }
06491     break;
06492   default:
06493     break;
06494   }
06495 
06496   // TODO: More simplifications are possible here.
06497 
06498   // Recursively simplify until we either hit a recursion limit or nothing
06499   // changes.
06500   if (Changed)
06501     return SimplifyICmpOperands(Pred, LHS, RHS, Depth+1);
06502 
06503   return Changed;
06504 
06505 trivially_true:
06506   // Return 0 == 0.
06507   LHS = RHS = getConstant(ConstantInt::getFalse(getContext()));
06508   Pred = ICmpInst::ICMP_EQ;
06509   return true;
06510 
06511 trivially_false:
06512   // Return 0 != 0.
06513   LHS = RHS = getConstant(ConstantInt::getFalse(getContext()));
06514   Pred = ICmpInst::ICMP_NE;
06515   return true;
06516 }
06517 
06518 bool ScalarEvolution::isKnownNegative(const SCEV *S) {
06519   return getSignedRange(S).getSignedMax().isNegative();
06520 }
06521 
06522 bool ScalarEvolution::isKnownPositive(const SCEV *S) {
06523   return getSignedRange(S).getSignedMin().isStrictlyPositive();
06524 }
06525 
06526 bool ScalarEvolution::isKnownNonNegative(const SCEV *S) {
06527   return !getSignedRange(S).getSignedMin().isNegative();
06528 }
06529 
06530 bool ScalarEvolution::isKnownNonPositive(const SCEV *S) {
06531   return !getSignedRange(S).getSignedMax().isStrictlyPositive();
06532 }
06533 
06534 bool ScalarEvolution::isKnownNonZero(const SCEV *S) {
06535   return isKnownNegative(S) || isKnownPositive(S);
06536 }
06537 
06538 bool ScalarEvolution::isKnownPredicate(ICmpInst::Predicate Pred,
06539                                        const SCEV *LHS, const SCEV *RHS) {
06540   // Canonicalize the inputs first.
06541   (void)SimplifyICmpOperands(Pred, LHS, RHS);
06542 
06543   // If LHS or RHS is an addrec, check to see if the condition is true in
06544   // every iteration of the loop.
06545   // If LHS and RHS are both addrec, both conditions must be true in
06546   // every iteration of the loop.
06547   const SCEVAddRecExpr *LAR = dyn_cast<SCEVAddRecExpr>(LHS);
06548   const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(RHS);
06549   bool LeftGuarded = false;
06550   bool RightGuarded = false;
06551   if (LAR) {
06552     const Loop *L = LAR->getLoop();
06553     if (isLoopEntryGuardedByCond(L, Pred, LAR->getStart(), RHS) &&
06554         isLoopBackedgeGuardedByCond(L, Pred, LAR->getPostIncExpr(*this), RHS)) {
06555       if (!RAR) return true;
06556       LeftGuarded = true;
06557     }
06558   }
06559   if (RAR) {
06560     const Loop *L = RAR->getLoop();
06561     if (isLoopEntryGuardedByCond(L, Pred, LHS, RAR->getStart()) &&
06562         isLoopBackedgeGuardedByCond(L, Pred, LHS, RAR->getPostIncExpr(*this))) {
06563       if (!LAR) return true;
06564       RightGuarded = true;
06565     }
06566   }
06567   if (LeftGuarded && RightGuarded)
06568     return true;
06569 
06570   // Otherwise see what can be done with known constant ranges.
06571   return isKnownPredicateWithRanges(Pred, LHS, RHS);
06572 }
06573 
06574 bool
06575 ScalarEvolution::isKnownPredicateWithRanges(ICmpInst::Predicate Pred,
06576                                             const SCEV *LHS, const SCEV *RHS) {
06577   if (HasSameValue(LHS, RHS))
06578     return ICmpInst::isTrueWhenEqual(Pred);
06579 
06580   // This code is split out from isKnownPredicate because it is called from
06581   // within isLoopEntryGuardedByCond.
06582   switch (Pred) {
06583   default:
06584     llvm_unreachable("Unexpected ICmpInst::Predicate value!");
06585   case ICmpInst::ICMP_SGT:
06586     std::swap(LHS, RHS);
06587   case ICmpInst::ICMP_SLT: {
06588     ConstantRange LHSRange = getSignedRange(LHS);
06589     ConstantRange RHSRange = getSignedRange(RHS);
06590     if (LHSRange.getSignedMax().slt(RHSRange.getSignedMin()))
06591       return true;
06592     if (LHSRange.getSignedMin().sge(RHSRange.getSignedMax()))
06593       return false;
06594     break;
06595   }
06596   case ICmpInst::ICMP_SGE:
06597     std::swap(LHS, RHS);
06598   case ICmpInst::ICMP_SLE: {
06599     ConstantRange LHSRange = getSignedRange(LHS);
06600     ConstantRange RHSRange = getSignedRange(RHS);
06601     if (LHSRange.getSignedMax().sle(RHSRange.getSignedMin()))
06602       return true;
06603     if (LHSRange.getSignedMin().sgt(RHSRange.getSignedMax()))
06604       return false;
06605     break;
06606   }
06607   case ICmpInst::ICMP_UGT:
06608     std::swap(LHS, RHS);
06609   case ICmpInst::ICMP_ULT: {
06610     ConstantRange LHSRange = getUnsignedRange(LHS);
06611     ConstantRange RHSRange = getUnsignedRange(RHS);
06612     if (LHSRange.getUnsignedMax().ult(RHSRange.getUnsignedMin()))
06613       return true;
06614     if (LHSRange.getUnsignedMin().uge(RHSRange.getUnsignedMax()))
06615       return false;
06616     break;
06617   }
06618   case ICmpInst::ICMP_UGE:
06619     std::swap(LHS, RHS);
06620   case ICmpInst::ICMP_ULE: {
06621     ConstantRange LHSRange = getUnsignedRange(LHS);
06622     ConstantRange RHSRange = getUnsignedRange(RHS);
06623     if (LHSRange.getUnsignedMax().ule(RHSRange.getUnsignedMin()))
06624       return true;
06625     if (LHSRange.getUnsignedMin().ugt(RHSRange.getUnsignedMax()))
06626       return false;
06627     break;
06628   }
06629   case ICmpInst::ICMP_NE: {
06630     if (getUnsignedRange(LHS).intersectWith(getUnsignedRange(RHS)).isEmptySet())
06631       return true;
06632     if (getSignedRange(LHS).intersectWith(getSignedRange(RHS)).isEmptySet())
06633       return true;
06634 
06635     const SCEV *Diff = getMinusSCEV(LHS, RHS);
06636     if (isKnownNonZero(Diff))
06637       return true;
06638     break;
06639   }
06640   case ICmpInst::ICMP_EQ:
06641     // The check at the top of the function catches the case where
06642     // the values are known to be equal.
06643     break;
06644   }
06645   return false;
06646 }
06647 
06648 /// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
06649 /// protected by a conditional between LHS and RHS.  This is used to
06650 /// to eliminate casts.
06651 bool
06652 ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
06653                                              ICmpInst::Predicate Pred,
06654                                              const SCEV *LHS, const SCEV *RHS) {
06655   // Interpret a null as meaning no loop, where there is obviously no guard
06656   // (interprocedural conditions notwithstanding).
06657   if (!L) return true;
06658 
06659   if (isKnownPredicateWithRanges(Pred, LHS, RHS)) return true;
06660 
06661   BasicBlock *Latch = L->getLoopLatch();
06662   if (!Latch)
06663     return false;
06664 
06665   BranchInst *LoopContinuePredicate =
06666     dyn_cast<BranchInst>(Latch->getTerminator());
06667   if (LoopContinuePredicate && LoopContinuePredicate->isConditional() &&
06668       isImpliedCond(Pred, LHS, RHS,
06669                     LoopContinuePredicate->getCondition(),
06670                     LoopContinuePredicate->getSuccessor(0) != L->getHeader()))
06671     return true;
06672 
06673   // Check conditions due to any @llvm.assume intrinsics.
06674   for (auto &CI : AT->assumptions(F)) {
06675     if (!DT->dominates(CI, Latch->getTerminator()))
06676       continue;
06677 
06678     if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
06679       return true;
06680   }
06681 
06682   return false;
06683 }
06684 
06685 /// isLoopEntryGuardedByCond - Test whether entry to the loop is protected
06686 /// by a conditional between LHS and RHS.  This is used to help avoid max
06687 /// expressions in loop trip counts, and to eliminate casts.
06688 bool
06689 ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L,
06690                                           ICmpInst::Predicate Pred,
06691                                           const SCEV *LHS, const SCEV *RHS) {
06692   // Interpret a null as meaning no loop, where there is obviously no guard
06693   // (interprocedural conditions notwithstanding).
06694   if (!L) return false;
06695 
06696   if (isKnownPredicateWithRanges(Pred, LHS, RHS)) return true;
06697 
06698   // Starting at the loop predecessor, climb up the predecessor chain, as long
06699   // as there are predecessors that can be found that have unique successors
06700   // leading to the original header.
06701   for (std::pair<BasicBlock *, BasicBlock *>
06702          Pair(L->getLoopPredecessor(), L->getHeader());
06703        Pair.first;
06704        Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
06705 
06706     BranchInst *LoopEntryPredicate =
06707       dyn_cast<BranchInst>(Pair.first->getTerminator());
06708     if (!LoopEntryPredicate ||
06709         LoopEntryPredicate->isUnconditional())
06710       continue;
06711 
06712     if (isImpliedCond(Pred, LHS, RHS,
06713                       LoopEntryPredicate->getCondition(),
06714                       LoopEntryPredicate->getSuccessor(0) != Pair.second))
06715       return true;
06716   }
06717 
06718   // Check conditions due to any @llvm.assume intrinsics.
06719   for (auto &CI : AT->assumptions(F)) {
06720     if (!DT->dominates(CI, L->getHeader()))
06721       continue;
06722 
06723     if (isImpliedCond(Pred, LHS, RHS, CI->getArgOperand(0), false))
06724       return true;
06725   }
06726 
06727   return false;
06728 }
06729 
06730 /// RAII wrapper to prevent recursive application of isImpliedCond.
06731 /// ScalarEvolution's PendingLoopPredicates set must be empty unless we are
06732 /// currently evaluating isImpliedCond.
06733 struct MarkPendingLoopPredicate {
06734   Value *Cond;
06735   DenseSet<Value*> &LoopPreds;
06736   bool Pending;
06737 
06738   MarkPendingLoopPredicate(Value *C, DenseSet<Value*> &LP)
06739     : Cond(C), LoopPreds(LP) {
06740     Pending = !LoopPreds.insert(Cond).second;
06741   }
06742   ~MarkPendingLoopPredicate() {
06743     if (!Pending)
06744       LoopPreds.erase(Cond);
06745   }
06746 };
06747 
06748 /// isImpliedCond - Test whether the condition described by Pred, LHS,
06749 /// and RHS is true whenever the given Cond value evaluates to true.
06750 bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred,
06751                                     const SCEV *LHS, const SCEV *RHS,
06752                                     Value *FoundCondValue,
06753                                     bool Inverse) {
06754   MarkPendingLoopPredicate Mark(FoundCondValue, PendingLoopPredicates);
06755   if (Mark.Pending)
06756     return false;
06757 
06758   // Recursively handle And and Or conditions.
06759   if (BinaryOperator *BO = dyn_cast<BinaryOperator>(FoundCondValue)) {
06760     if (BO->getOpcode() == Instruction::And) {
06761       if (!Inverse)
06762         return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) ||
06763                isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse);
06764     } else if (BO->getOpcode() == Instruction::Or) {
06765       if (Inverse)
06766         return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) ||
06767                isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse);
06768     }
06769   }
06770 
06771   ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue);
06772   if (!ICI) return false;
06773 
06774   // Bail if the ICmp's operands' types are wider than the needed type
06775   // before attempting to call getSCEV on them. This avoids infinite
06776   // recursion, since the analysis of widening casts can require loop
06777   // exit condition information for overflow checking, which would
06778   // lead back here.
06779   if (getTypeSizeInBits(LHS->getType()) <
06780       getTypeSizeInBits(ICI->getOperand(0)->getType()))
06781     return false;
06782 
06783   // Now that we found a conditional branch that dominates the loop or controls
06784   // the loop latch. Check to see if it is the comparison we are looking for.
06785   ICmpInst::Predicate FoundPred;
06786   if (Inverse)
06787     FoundPred = ICI->getInversePredicate();
06788   else
06789     FoundPred = ICI->getPredicate();
06790 
06791   const SCEV *FoundLHS = getSCEV(ICI->getOperand(0));
06792   const SCEV *FoundRHS = getSCEV(ICI->getOperand(1));
06793 
06794   // Balance the types. The case where FoundLHS' type is wider than
06795   // LHS' type is checked for above.
06796   if (getTypeSizeInBits(LHS->getType()) >
06797       getTypeSizeInBits(FoundLHS->getType())) {
06798     if (CmpInst::isSigned(FoundPred)) {
06799       FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType());
06800       FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType());
06801     } else {
06802       FoundLHS = getZeroExtendExpr(FoundLHS, LHS->getType());
06803       FoundRHS = getZeroExtendExpr(FoundRHS, LHS->getType());
06804     }
06805   }
06806 
06807   // Canonicalize the query to match the way instcombine will have
06808   // canonicalized the comparison.
06809   if (SimplifyICmpOperands(Pred, LHS, RHS))
06810     if (LHS == RHS)
06811       return CmpInst::isTrueWhenEqual(Pred);
06812   if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS))
06813     if (FoundLHS == FoundRHS)
06814       return CmpInst::isFalseWhenEqual(FoundPred);
06815 
06816   // Check to see if we can make the LHS or RHS match.
06817   if (LHS == FoundRHS || RHS == FoundLHS) {
06818     if (isa<SCEVConstant>(RHS)) {
06819       std::swap(FoundLHS, FoundRHS);
06820       FoundPred = ICmpInst::getSwappedPredicate(FoundPred);
06821     } else {
06822       std::swap(LHS, RHS);
06823       Pred = ICmpInst::getSwappedPredicate(Pred);
06824     }
06825   }
06826 
06827   // Check whether the found predicate is the same as the desired predicate.
06828   if (FoundPred == Pred)
06829     return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS);
06830 
06831   // Check whether swapping the found predicate makes it the same as the
06832   // desired predicate.
06833   if (ICmpInst::getSwappedPredicate(FoundPred) == Pred) {
06834     if (isa<SCEVConstant>(RHS))
06835       return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS);
06836     else
06837       return isImpliedCondOperands(ICmpInst::getSwappedPredicate(Pred),
06838                                    RHS, LHS, FoundLHS, FoundRHS);
06839   }
06840 
06841   // Check if we can make progress by sharpening ranges.
06842   if (FoundPred == ICmpInst::ICMP_NE &&
06843       (isa<SCEVConstant>(FoundLHS) || isa<SCEVConstant>(FoundRHS))) {
06844 
06845     const SCEVConstant *C = nullptr;
06846     const SCEV *V = nullptr;
06847 
06848     if (isa<SCEVConstant>(FoundLHS)) {
06849       C = cast<SCEVConstant>(FoundLHS);
06850       V = FoundRHS;
06851     } else {
06852       C = cast<SCEVConstant>(FoundRHS);
06853       V = FoundLHS;
06854     }
06855 
06856     // The guarding predicate tells us that C != V. If the known range
06857     // of V is [C, t), we can sharpen the range to [C + 1, t).  The
06858     // range we consider has to correspond to same signedness as the
06859     // predicate we're interested in folding.
06860 
06861     APInt Min = ICmpInst::isSigned(Pred) ?
06862         getSignedRange(V).getSignedMin() : getUnsignedRange(V).getUnsignedMin();
06863 
06864     if (Min == C->getValue()->getValue()) {
06865       // Given (V >= Min && V != Min) we conclude V >= (Min + 1).
06866       // This is true even if (Min + 1) wraps around -- in case of
06867       // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)).
06868 
06869       APInt SharperMin = Min + 1;
06870 
06871       switch (Pred) {
06872         case ICmpInst::ICMP_SGE:
06873         case ICmpInst::ICMP_UGE:
06874           // We know V `Pred` SharperMin.  If this implies LHS `Pred`
06875           // RHS, we're done.
06876           if (isImpliedCondOperands(Pred, LHS, RHS, V,
06877                                     getConstant(SharperMin)))
06878             return true;
06879 
06880         case ICmpInst::ICMP_SGT:
06881         case ICmpInst::ICMP_UGT:
06882           // We know from the range information that (V `Pred` Min ||
06883           // V == Min).  We know from the guarding condition that !(V
06884           // == Min).  This gives us
06885           //
06886           //       V `Pred` Min || V == Min && !(V == Min)
06887           //   =>  V `Pred` Min
06888           //
06889           // If V `Pred` Min implies LHS `Pred` RHS, we're done.
06890 
06891           if (isImpliedCondOperands(Pred, LHS, RHS, V, getConstant(Min)))
06892             return true;
06893 
06894         default:
06895           // No change
06896           break;
06897       }
06898     }
06899   }
06900 
06901   // Check whether the actual condition is beyond sufficient.
06902   if (FoundPred == ICmpInst::ICMP_EQ)
06903     if (ICmpInst::isTrueWhenEqual(Pred))
06904       if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS))
06905         return true;
06906   if (Pred == ICmpInst::ICMP_NE)
06907     if (!ICmpInst::isTrueWhenEqual(FoundPred))
06908       if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS))
06909         return true;
06910 
06911   // Otherwise assume the worst.
06912   return false;
06913 }
06914 
06915 /// isImpliedCondOperands - Test whether the condition described by Pred,
06916 /// LHS, and RHS is true whenever the condition described by Pred, FoundLHS,
06917 /// and FoundRHS is true.
06918 bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred,
06919                                             const SCEV *LHS, const SCEV *RHS,
06920                                             const SCEV *FoundLHS,
06921                                             const SCEV *FoundRHS) {
06922   return isImpliedCondOperandsHelper(Pred, LHS, RHS,
06923                                      FoundLHS, FoundRHS) ||
06924          // ~x < ~y --> x > y
06925          isImpliedCondOperandsHelper(Pred, LHS, RHS,
06926                                      getNotSCEV(FoundRHS),
06927                                      getNotSCEV(FoundLHS));
06928 }
06929 
06930 /// isImpliedCondOperandsHelper - Test whether the condition described by
06931 /// Pred, LHS, and RHS is true whenever the condition described by Pred,
06932 /// FoundLHS, and FoundRHS is true.
06933 bool
06934 ScalarEvolution::isImpliedCondOperandsHelper(ICmpInst::Predicate Pred,
06935                                              const SCEV *LHS, const SCEV *RHS,
06936                                              const SCEV *FoundLHS,
06937                                              const SCEV *FoundRHS) {
06938   switch (Pred) {
06939   default: llvm_unreachable("Unexpected ICmpInst::Predicate value!");
06940   case ICmpInst::ICMP_EQ:
06941   case ICmpInst::ICMP_NE:
06942     if (HasSameValue(LHS, FoundLHS) && HasSameValue(RHS, FoundRHS))
06943       return true;
06944     break;
06945   case ICmpInst::ICMP_SLT:
06946   case ICmpInst::ICMP_SLE:
06947     if (isKnownPredicateWithRanges(ICmpInst::ICMP_SLE, LHS, FoundLHS) &&
06948         isKnownPredicateWithRanges(ICmpInst::ICMP_SGE, RHS, FoundRHS))
06949       return true;
06950     break;
06951   case ICmpInst::ICMP_SGT:
06952   case ICmpInst::ICMP_SGE:
06953     if (isKnownPredicateWithRanges(ICmpInst::ICMP_SGE, LHS, FoundLHS) &&
06954         isKnownPredicateWithRanges(ICmpInst::ICMP_SLE, RHS, FoundRHS))
06955       return true;
06956     break;
06957   case ICmpInst::ICMP_ULT:
06958   case ICmpInst::ICMP_ULE:
06959     if (isKnownPredicateWithRanges(ICmpInst::ICMP_ULE, LHS, FoundLHS) &&
06960         isKnownPredicateWithRanges(