LLVM API Documentation

ScalarEvolution.cpp
Go to the documentation of this file.
00001 //===- ScalarEvolution.cpp - Scalar Evolution Analysis ----------*- C++ -*-===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This file contains the implementation of the scalar evolution analysis
00011 // engine, which is used primarily to analyze expressions involving induction
00012 // variables in loops.
00013 //
00014 // There are several aspects to this library.  First is the representation of
00015 // scalar expressions, which are represented as subclasses of the SCEV class.
00016 // These classes are used to represent certain types of subexpressions that we
00017 // can handle. We only create one SCEV of a particular shape, so
00018 // pointer-comparisons for equality are legal.
00019 //
00020 // One important aspect of the SCEV objects is that they are never cyclic, even
00021 // if there is a cycle in the dataflow for an expression (ie, a PHI node).  If
00022 // the PHI node is one of the idioms that we can represent (e.g., a polynomial
00023 // recurrence) then we represent it directly as a recurrence node, otherwise we
00024 // represent it as a SCEVUnknown node.
00025 //
00026 // In addition to being able to represent expressions of various types, we also
00027 // have folders that are used to build the *canonical* representation for a
00028 // particular expression.  These folders are capable of using a variety of
00029 // rewrite rules to simplify the expressions.
00030 //
00031 // Once the folders are defined, we can implement the more interesting
00032 // higher-level code, such as the code that recognizes PHI nodes of various
00033 // types, computes the execution count of a loop, etc.
00034 //
00035 // TODO: We should use these routines and value representations to implement
00036 // dependence analysis!
00037 //
00038 //===----------------------------------------------------------------------===//
00039 //
00040 // There are several good references for the techniques used in this analysis.
00041 //
00042 //  Chains of recurrences -- a method to expedite the evaluation
00043 //  of closed-form functions
00044 //  Olaf Bachmann, Paul S. Wang, Eugene V. Zima
00045 //
00046 //  On computational properties of chains of recurrences
00047 //  Eugene V. Zima
00048 //
00049 //  Symbolic Evaluation of Chains of Recurrences for Loop Optimization
00050 //  Robert A. van Engelen
00051 //
00052 //  Efficient Symbolic Analysis for Optimizing Compilers
00053 //  Robert A. van Engelen
00054 //
00055 //  Using the chains of recurrences algebra for data dependence testing and
00056 //  induction variable substitution
00057 //  MS Thesis, Johnie Birch
00058 //
00059 //===----------------------------------------------------------------------===//
00060 
00061 #define DEBUG_TYPE "scalar-evolution"
00062 #include "llvm/Analysis/ScalarEvolution.h"
00063 #include "llvm/ADT/STLExtras.h"
00064 #include "llvm/ADT/SmallPtrSet.h"
00065 #include "llvm/ADT/Statistic.h"
00066 #include "llvm/Analysis/ConstantFolding.h"
00067 #include "llvm/Analysis/InstructionSimplify.h"
00068 #include "llvm/Analysis/LoopInfo.h"
00069 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
00070 #include "llvm/Analysis/ValueTracking.h"
00071 #include "llvm/IR/ConstantRange.h"
00072 #include "llvm/IR/Constants.h"
00073 #include "llvm/IR/DataLayout.h"
00074 #include "llvm/IR/DerivedTypes.h"
00075 #include "llvm/IR/Dominators.h"
00076 #include "llvm/IR/GetElementPtrTypeIterator.h"
00077 #include "llvm/IR/GlobalAlias.h"
00078 #include "llvm/IR/GlobalVariable.h"
00079 #include "llvm/IR/InstIterator.h"
00080 #include "llvm/IR/Instructions.h"
00081 #include "llvm/IR/LLVMContext.h"
00082 #include "llvm/IR/Operator.h"
00083 #include "llvm/Support/CommandLine.h"
00084 #include "llvm/Support/Debug.h"
00085 #include "llvm/Support/ErrorHandling.h"
00086 #include "llvm/Support/MathExtras.h"
00087 #include "llvm/Support/raw_ostream.h"
00088 #include "llvm/Target/TargetLibraryInfo.h"
00089 #include <algorithm>
00090 using namespace llvm;
00091 
00092 STATISTIC(NumArrayLenItCounts,
00093           "Number of trip counts computed with array length");
00094 STATISTIC(NumTripCountsComputed,
00095           "Number of loops with predictable loop counts");
00096 STATISTIC(NumTripCountsNotComputed,
00097           "Number of loops without predictable loop counts");
00098 STATISTIC(NumBruteForceTripCountsComputed,
00099           "Number of loops with trip counts computed by force");
00100 
00101 static cl::opt<unsigned>
00102 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
00103                         cl::desc("Maximum number of iterations SCEV will "
00104                                  "symbolically execute a constant "
00105                                  "derived loop"),
00106                         cl::init(100));
00107 
00108 // FIXME: Enable this with XDEBUG when the test suite is clean.
00109 static cl::opt<bool>
00110 VerifySCEV("verify-scev",
00111            cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
00112 
00113 INITIALIZE_PASS_BEGIN(ScalarEvolution, "scalar-evolution",
00114                 "Scalar Evolution Analysis", false, true)
00115 INITIALIZE_PASS_DEPENDENCY(LoopInfo)
00116 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
00117 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfo)
00118 INITIALIZE_PASS_END(ScalarEvolution, "scalar-evolution",
00119                 "Scalar Evolution Analysis", false, true)
00120 char ScalarEvolution::ID = 0;
00121 
00122 //===----------------------------------------------------------------------===//
00123 //                           SCEV class definitions
00124 //===----------------------------------------------------------------------===//
00125 
00126 //===----------------------------------------------------------------------===//
00127 // Implementation of the SCEV class.
00128 //
00129 
00130 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
00131 void SCEV::dump() const {
00132   print(dbgs());
00133   dbgs() << '\n';
00134 }
00135 #endif
00136 
00137 void SCEV::print(raw_ostream &OS) const {
00138   switch (static_cast<SCEVTypes>(getSCEVType())) {
00139   case scConstant:
00140     cast<SCEVConstant>(this)->getValue()->printAsOperand(OS, false);
00141     return;
00142   case scTruncate: {
00143     const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this);
00144     const SCEV *Op = Trunc->getOperand();
00145     OS << "(trunc " << *Op->getType() << " " << *Op << " to "
00146        << *Trunc->getType() << ")";
00147     return;
00148   }
00149   case scZeroExtend: {
00150     const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(this);
00151     const SCEV *Op = ZExt->getOperand();
00152     OS << "(zext " << *Op->getType() << " " << *Op << " to "
00153        << *ZExt->getType() << ")";
00154     return;
00155   }
00156   case scSignExtend: {
00157     const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(this);
00158     const SCEV *Op = SExt->getOperand();
00159     OS << "(sext " << *Op->getType() << " " << *Op << " to "
00160        << *SExt->getType() << ")";
00161     return;
00162   }
00163   case scAddRecExpr: {
00164     const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(this);
00165     OS << "{" << *AR->getOperand(0);
00166     for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
00167       OS << ",+," << *AR->getOperand(i);
00168     OS << "}<";
00169     if (AR->getNoWrapFlags(FlagNUW))
00170       OS << "nuw><";
00171     if (AR->getNoWrapFlags(FlagNSW))
00172       OS << "nsw><";
00173     if (AR->getNoWrapFlags(FlagNW) &&
00174         !AR->getNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW)))
00175       OS << "nw><";
00176     AR->getLoop()->getHeader()->printAsOperand(OS, /*PrintType=*/false);
00177     OS << ">";
00178     return;
00179   }
00180   case scAddExpr:
00181   case scMulExpr:
00182   case scUMaxExpr:
00183   case scSMaxExpr: {
00184     const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(this);
00185     const char *OpStr = nullptr;
00186     switch (NAry->getSCEVType()) {
00187     case scAddExpr: OpStr = " + "; break;
00188     case scMulExpr: OpStr = " * "; break;
00189     case scUMaxExpr: OpStr = " umax "; break;
00190     case scSMaxExpr: OpStr = " smax "; break;
00191     }
00192     OS << "(";
00193     for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end();
00194          I != E; ++I) {
00195       OS << **I;
00196       if (std::next(I) != E)
00197         OS << OpStr;
00198     }
00199     OS << ")";
00200     switch (NAry->getSCEVType()) {
00201     case scAddExpr:
00202     case scMulExpr:
00203       if (NAry->getNoWrapFlags(FlagNUW))
00204         OS << "<nuw>";
00205       if (NAry->getNoWrapFlags(FlagNSW))
00206         OS << "<nsw>";
00207     }
00208     return;
00209   }
00210   case scUDivExpr: {
00211     const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this);
00212     OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")";
00213     return;
00214   }
00215   case scUnknown: {
00216     const SCEVUnknown *U = cast<SCEVUnknown>(this);
00217     Type *AllocTy;
00218     if (U->isSizeOf(AllocTy)) {
00219       OS << "sizeof(" << *AllocTy << ")";
00220       return;
00221     }
00222     if (U->isAlignOf(AllocTy)) {
00223       OS << "alignof(" << *AllocTy << ")";
00224       return;
00225     }
00226 
00227     Type *CTy;
00228     Constant *FieldNo;
00229     if (U->isOffsetOf(CTy, FieldNo)) {
00230       OS << "offsetof(" << *CTy << ", ";
00231       FieldNo->printAsOperand(OS, false);
00232       OS << ")";
00233       return;
00234     }
00235 
00236     // Otherwise just print it normally.
00237     U->getValue()->printAsOperand(OS, false);
00238     return;
00239   }
00240   case scCouldNotCompute:
00241     OS << "***COULDNOTCOMPUTE***";
00242     return;
00243   }
00244   llvm_unreachable("Unknown SCEV kind!");
00245 }
00246 
00247 Type *SCEV::getType() const {
00248   switch (static_cast<SCEVTypes>(getSCEVType())) {
00249   case scConstant:
00250     return cast<SCEVConstant>(this)->getType();
00251   case scTruncate:
00252   case scZeroExtend:
00253   case scSignExtend:
00254     return cast<SCEVCastExpr>(this)->getType();
00255   case scAddRecExpr:
00256   case scMulExpr:
00257   case scUMaxExpr:
00258   case scSMaxExpr:
00259     return cast<SCEVNAryExpr>(this)->getType();
00260   case scAddExpr:
00261     return cast<SCEVAddExpr>(this)->getType();
00262   case scUDivExpr:
00263     return cast<SCEVUDivExpr>(this)->getType();
00264   case scUnknown:
00265     return cast<SCEVUnknown>(this)->getType();
00266   case scCouldNotCompute:
00267     llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
00268   }
00269   llvm_unreachable("Unknown SCEV kind!");
00270 }
00271 
00272 bool SCEV::isZero() const {
00273   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
00274     return SC->getValue()->isZero();
00275   return false;
00276 }
00277 
00278 bool SCEV::isOne() const {
00279   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
00280     return SC->getValue()->isOne();
00281   return false;
00282 }
00283 
00284 bool SCEV::isAllOnesValue() const {
00285   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(this))
00286     return SC->getValue()->isAllOnesValue();
00287   return false;
00288 }
00289 
00290 /// isNonConstantNegative - Return true if the specified scev is negated, but
00291 /// not a constant.
00292 bool SCEV::isNonConstantNegative() const {
00293   const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(this);
00294   if (!Mul) return false;
00295 
00296   // If there is a constant factor, it will be first.
00297   const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0));
00298   if (!SC) return false;
00299 
00300   // Return true if the value is negative, this matches things like (-42 * V).
00301   return SC->getValue()->getValue().isNegative();
00302 }
00303 
00304 SCEVCouldNotCompute::SCEVCouldNotCompute() :
00305   SCEV(FoldingSetNodeIDRef(), scCouldNotCompute) {}
00306 
00307 bool SCEVCouldNotCompute::classof(const SCEV *S) {
00308   return S->getSCEVType() == scCouldNotCompute;
00309 }
00310 
00311 const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
00312   FoldingSetNodeID ID;
00313   ID.AddInteger(scConstant);
00314   ID.AddPointer(V);
00315   void *IP = nullptr;
00316   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
00317   SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V);
00318   UniqueSCEVs.InsertNode(S, IP);
00319   return S;
00320 }
00321 
00322 const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
00323   return getConstant(ConstantInt::get(getContext(), Val));
00324 }
00325 
00326 const SCEV *
00327 ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
00328   IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty));
00329   return getConstant(ConstantInt::get(ITy, V, isSigned));
00330 }
00331 
00332 SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID,
00333                            unsigned SCEVTy, const SCEV *op, Type *ty)
00334   : SCEV(ID, SCEVTy), Op(op), Ty(ty) {}
00335 
00336 SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID,
00337                                    const SCEV *op, Type *ty)
00338   : SCEVCastExpr(ID, scTruncate, op, ty) {
00339   assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
00340          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
00341          "Cannot truncate non-integer value!");
00342 }
00343 
00344 SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID,
00345                                        const SCEV *op, Type *ty)
00346   : SCEVCastExpr(ID, scZeroExtend, op, ty) {
00347   assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
00348          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
00349          "Cannot zero extend non-integer value!");
00350 }
00351 
00352 SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID,
00353                                        const SCEV *op, Type *ty)
00354   : SCEVCastExpr(ID, scSignExtend, op, ty) {
00355   assert((Op->getType()->isIntegerTy() || Op->getType()->isPointerTy()) &&
00356          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
00357          "Cannot sign extend non-integer value!");
00358 }
00359 
00360 void SCEVUnknown::deleted() {
00361   // Clear this SCEVUnknown from various maps.
00362   SE->forgetMemoizedResults(this);
00363 
00364   // Remove this SCEVUnknown from the uniquing map.
00365   SE->UniqueSCEVs.RemoveNode(this);
00366 
00367   // Release the value.
00368   setValPtr(nullptr);
00369 }
00370 
00371 void SCEVUnknown::allUsesReplacedWith(Value *New) {
00372   // Clear this SCEVUnknown from various maps.
00373   SE->forgetMemoizedResults(this);
00374 
00375   // Remove this SCEVUnknown from the uniquing map.
00376   SE->UniqueSCEVs.RemoveNode(this);
00377 
00378   // Update this SCEVUnknown to point to the new value. This is needed
00379   // because there may still be outstanding SCEVs which still point to
00380   // this SCEVUnknown.
00381   setValPtr(New);
00382 }
00383 
00384 bool SCEVUnknown::isSizeOf(Type *&AllocTy) const {
00385   if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
00386     if (VCE->getOpcode() == Instruction::PtrToInt)
00387       if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
00388         if (CE->getOpcode() == Instruction::GetElementPtr &&
00389             CE->getOperand(0)->isNullValue() &&
00390             CE->getNumOperands() == 2)
00391           if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(1)))
00392             if (CI->isOne()) {
00393               AllocTy = cast<PointerType>(CE->getOperand(0)->getType())
00394                                  ->getElementType();
00395               return true;
00396             }
00397 
00398   return false;
00399 }
00400 
00401 bool SCEVUnknown::isAlignOf(Type *&AllocTy) const {
00402   if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
00403     if (VCE->getOpcode() == Instruction::PtrToInt)
00404       if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
00405         if (CE->getOpcode() == Instruction::GetElementPtr &&
00406             CE->getOperand(0)->isNullValue()) {
00407           Type *Ty =
00408             cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
00409           if (StructType *STy = dyn_cast<StructType>(Ty))
00410             if (!STy->isPacked() &&
00411                 CE->getNumOperands() == 3 &&
00412                 CE->getOperand(1)->isNullValue()) {
00413               if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(2)))
00414                 if (CI->isOne() &&
00415                     STy->getNumElements() == 2 &&
00416                     STy->getElementType(0)->isIntegerTy(1)) {
00417                   AllocTy = STy->getElementType(1);
00418                   return true;
00419                 }
00420             }
00421         }
00422 
00423   return false;
00424 }
00425 
00426 bool SCEVUnknown::isOffsetOf(Type *&CTy, Constant *&FieldNo) const {
00427   if (ConstantExpr *VCE = dyn_cast<ConstantExpr>(getValue()))
00428     if (VCE->getOpcode() == Instruction::PtrToInt)
00429       if (ConstantExpr *CE = dyn_cast<ConstantExpr>(VCE->getOperand(0)))
00430         if (CE->getOpcode() == Instruction::GetElementPtr &&
00431             CE->getNumOperands() == 3 &&
00432             CE->getOperand(0)->isNullValue() &&
00433             CE->getOperand(1)->isNullValue()) {
00434           Type *Ty =
00435             cast<PointerType>(CE->getOperand(0)->getType())->getElementType();
00436           // Ignore vector types here so that ScalarEvolutionExpander doesn't
00437           // emit getelementptrs that index into vectors.
00438           if (Ty->isStructTy() || Ty->isArrayTy()) {
00439             CTy = Ty;
00440             FieldNo = CE->getOperand(2);
00441             return true;
00442           }
00443         }
00444 
00445   return false;
00446 }
00447 
00448 //===----------------------------------------------------------------------===//
00449 //                               SCEV Utilities
00450 //===----------------------------------------------------------------------===//
00451 
00452 namespace {
00453   /// SCEVComplexityCompare - Return true if the complexity of the LHS is less
00454   /// than the complexity of the RHS.  This comparator is used to canonicalize
00455   /// expressions.
00456   class SCEVComplexityCompare {
00457     const LoopInfo *const LI;
00458   public:
00459     explicit SCEVComplexityCompare(const LoopInfo *li) : LI(li) {}
00460 
00461     // Return true or false if LHS is less than, or at least RHS, respectively.
00462     bool operator()(const SCEV *LHS, const SCEV *RHS) const {
00463       return compare(LHS, RHS) < 0;
00464     }
00465 
00466     // Return negative, zero, or positive, if LHS is less than, equal to, or
00467     // greater than RHS, respectively. A three-way result allows recursive
00468     // comparisons to be more efficient.
00469     int compare(const SCEV *LHS, const SCEV *RHS) const {
00470       // Fast-path: SCEVs are uniqued so we can do a quick equality check.
00471       if (LHS == RHS)
00472         return 0;
00473 
00474       // Primarily, sort the SCEVs by their getSCEVType().
00475       unsigned LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
00476       if (LType != RType)
00477         return (int)LType - (int)RType;
00478 
00479       // Aside from the getSCEVType() ordering, the particular ordering
00480       // isn't very important except that it's beneficial to be consistent,
00481       // so that (a + b) and (b + a) don't end up as different expressions.
00482       switch (static_cast<SCEVTypes>(LType)) {
00483       case scUnknown: {
00484         const SCEVUnknown *LU = cast<SCEVUnknown>(LHS);
00485         const SCEVUnknown *RU = cast<SCEVUnknown>(RHS);
00486 
00487         // Sort SCEVUnknown values with some loose heuristics. TODO: This is
00488         // not as complete as it could be.
00489         const Value *LV = LU->getValue(), *RV = RU->getValue();
00490 
00491         // Order pointer values after integer values. This helps SCEVExpander
00492         // form GEPs.
00493         bool LIsPointer = LV->getType()->isPointerTy(),
00494              RIsPointer = RV->getType()->isPointerTy();
00495         if (LIsPointer != RIsPointer)
00496           return (int)LIsPointer - (int)RIsPointer;
00497 
00498         // Compare getValueID values.
00499         unsigned LID = LV->getValueID(),
00500                  RID = RV->getValueID();
00501         if (LID != RID)
00502           return (int)LID - (int)RID;
00503 
00504         // Sort arguments by their position.
00505         if (const Argument *LA = dyn_cast<Argument>(LV)) {
00506           const Argument *RA = cast<Argument>(RV);
00507           unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
00508           return (int)LArgNo - (int)RArgNo;
00509         }
00510 
00511         // For instructions, compare their loop depth, and their operand
00512         // count.  This is pretty loose.
00513         if (const Instruction *LInst = dyn_cast<Instruction>(LV)) {
00514           const Instruction *RInst = cast<Instruction>(RV);
00515 
00516           // Compare loop depths.
00517           const BasicBlock *LParent = LInst->getParent(),
00518                            *RParent = RInst->getParent();
00519           if (LParent != RParent) {
00520             unsigned LDepth = LI->getLoopDepth(LParent),
00521                      RDepth = LI->getLoopDepth(RParent);
00522             if (LDepth != RDepth)
00523               return (int)LDepth - (int)RDepth;
00524           }
00525 
00526           // Compare the number of operands.
00527           unsigned LNumOps = LInst->getNumOperands(),
00528                    RNumOps = RInst->getNumOperands();
00529           return (int)LNumOps - (int)RNumOps;
00530         }
00531 
00532         return 0;
00533       }
00534 
00535       case scConstant: {
00536         const SCEVConstant *LC = cast<SCEVConstant>(LHS);
00537         const SCEVConstant *RC = cast<SCEVConstant>(RHS);
00538 
00539         // Compare constant values.
00540         const APInt &LA = LC->getValue()->getValue();
00541         const APInt &RA = RC->getValue()->getValue();
00542         unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
00543         if (LBitWidth != RBitWidth)
00544           return (int)LBitWidth - (int)RBitWidth;
00545         return LA.ult(RA) ? -1 : 1;
00546       }
00547 
00548       case scAddRecExpr: {
00549         const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(LHS);
00550         const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(RHS);
00551 
00552         // Compare addrec loop depths.
00553         const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
00554         if (LLoop != RLoop) {
00555           unsigned LDepth = LLoop->getLoopDepth(),
00556                    RDepth = RLoop->getLoopDepth();
00557           if (LDepth != RDepth)
00558             return (int)LDepth - (int)RDepth;
00559         }
00560 
00561         // Addrec complexity grows with operand count.
00562         unsigned LNumOps = LA->getNumOperands(), RNumOps = RA->getNumOperands();
00563         if (LNumOps != RNumOps)
00564           return (int)LNumOps - (int)RNumOps;
00565 
00566         // Lexicographically compare.
00567         for (unsigned i = 0; i != LNumOps; ++i) {
00568           long X = compare(LA->getOperand(i), RA->getOperand(i));
00569           if (X != 0)
00570             return X;
00571         }
00572 
00573         return 0;
00574       }
00575 
00576       case scAddExpr:
00577       case scMulExpr:
00578       case scSMaxExpr:
00579       case scUMaxExpr: {
00580         const SCEVNAryExpr *LC = cast<SCEVNAryExpr>(LHS);
00581         const SCEVNAryExpr *RC = cast<SCEVNAryExpr>(RHS);
00582 
00583         // Lexicographically compare n-ary expressions.
00584         unsigned LNumOps = LC->getNumOperands(), RNumOps = RC->getNumOperands();
00585         if (LNumOps != RNumOps)
00586           return (int)LNumOps - (int)RNumOps;
00587 
00588         for (unsigned i = 0; i != LNumOps; ++i) {
00589           if (i >= RNumOps)
00590             return 1;
00591           long X = compare(LC->getOperand(i), RC->getOperand(i));
00592           if (X != 0)
00593             return X;
00594         }
00595         return (int)LNumOps - (int)RNumOps;
00596       }
00597 
00598       case scUDivExpr: {
00599         const SCEVUDivExpr *LC = cast<SCEVUDivExpr>(LHS);
00600         const SCEVUDivExpr *RC = cast<SCEVUDivExpr>(RHS);
00601 
00602         // Lexicographically compare udiv expressions.
00603         long X = compare(LC->getLHS(), RC->getLHS());
00604         if (X != 0)
00605           return X;
00606         return compare(LC->getRHS(), RC->getRHS());
00607       }
00608 
00609       case scTruncate:
00610       case scZeroExtend:
00611       case scSignExtend: {
00612         const SCEVCastExpr *LC = cast<SCEVCastExpr>(LHS);
00613         const SCEVCastExpr *RC = cast<SCEVCastExpr>(RHS);
00614 
00615         // Compare cast expressions by operand.
00616         return compare(LC->getOperand(), RC->getOperand());
00617       }
00618 
00619       case scCouldNotCompute:
00620         llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
00621       }
00622       llvm_unreachable("Unknown SCEV kind!");
00623     }
00624   };
00625 }
00626 
00627 /// GroupByComplexity - Given a list of SCEV objects, order them by their
00628 /// complexity, and group objects of the same complexity together by value.
00629 /// When this routine is finished, we know that any duplicates in the vector are
00630 /// consecutive and that complexity is monotonically increasing.
00631 ///
00632 /// Note that we go take special precautions to ensure that we get deterministic
00633 /// results from this routine.  In other words, we don't want the results of
00634 /// this to depend on where the addresses of various SCEV objects happened to
00635 /// land in memory.
00636 ///
00637 static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops,
00638                               LoopInfo *LI) {
00639   if (Ops.size() < 2) return;  // Noop
00640   if (Ops.size() == 2) {
00641     // This is the common case, which also happens to be trivially simple.
00642     // Special case it.
00643     const SCEV *&LHS = Ops[0], *&RHS = Ops[1];
00644     if (SCEVComplexityCompare(LI)(RHS, LHS))
00645       std::swap(LHS, RHS);
00646     return;
00647   }
00648 
00649   // Do the rough sort by complexity.
00650   std::stable_sort(Ops.begin(), Ops.end(), SCEVComplexityCompare(LI));
00651 
00652   // Now that we are sorted by complexity, group elements of the same
00653   // complexity.  Note that this is, at worst, N^2, but the vector is likely to
00654   // be extremely short in practice.  Note that we take this approach because we
00655   // do not want to depend on the addresses of the objects we are grouping.
00656   for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
00657     const SCEV *S = Ops[i];
00658     unsigned Complexity = S->getSCEVType();
00659 
00660     // If there are any objects of the same complexity and same value as this
00661     // one, group them.
00662     for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
00663       if (Ops[j] == S) { // Found a duplicate.
00664         // Move it to immediately after i'th element.
00665         std::swap(Ops[i+1], Ops[j]);
00666         ++i;   // no need to rescan it.
00667         if (i == e-2) return;  // Done!
00668       }
00669     }
00670   }
00671 }
00672 
00673 
00674 
00675 //===----------------------------------------------------------------------===//
00676 //                      Simple SCEV method implementations
00677 //===----------------------------------------------------------------------===//
00678 
00679 /// BinomialCoefficient - Compute BC(It, K).  The result has width W.
00680 /// Assume, K > 0.
00681 static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
00682                                        ScalarEvolution &SE,
00683                                        Type *ResultTy) {
00684   // Handle the simplest case efficiently.
00685   if (K == 1)
00686     return SE.getTruncateOrZeroExtend(It, ResultTy);
00687 
00688   // We are using the following formula for BC(It, K):
00689   //
00690   //   BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
00691   //
00692   // Suppose, W is the bitwidth of the return value.  We must be prepared for
00693   // overflow.  Hence, we must assure that the result of our computation is
00694   // equal to the accurate one modulo 2^W.  Unfortunately, division isn't
00695   // safe in modular arithmetic.
00696   //
00697   // However, this code doesn't use exactly that formula; the formula it uses
00698   // is something like the following, where T is the number of factors of 2 in
00699   // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
00700   // exponentiation:
00701   //
00702   //   BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
00703   //
00704   // This formula is trivially equivalent to the previous formula.  However,
00705   // this formula can be implemented much more efficiently.  The trick is that
00706   // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
00707   // arithmetic.  To do exact division in modular arithmetic, all we have
00708   // to do is multiply by the inverse.  Therefore, this step can be done at
00709   // width W.
00710   //
00711   // The next issue is how to safely do the division by 2^T.  The way this
00712   // is done is by doing the multiplication step at a width of at least W + T
00713   // bits.  This way, the bottom W+T bits of the product are accurate. Then,
00714   // when we perform the division by 2^T (which is equivalent to a right shift
00715   // by T), the bottom W bits are accurate.  Extra bits are okay; they'll get
00716   // truncated out after the division by 2^T.
00717   //
00718   // In comparison to just directly using the first formula, this technique
00719   // is much more efficient; using the first formula requires W * K bits,
00720   // but this formula less than W + K bits. Also, the first formula requires
00721   // a division step, whereas this formula only requires multiplies and shifts.
00722   //
00723   // It doesn't matter whether the subtraction step is done in the calculation
00724   // width or the input iteration count's width; if the subtraction overflows,
00725   // the result must be zero anyway.  We prefer here to do it in the width of
00726   // the induction variable because it helps a lot for certain cases; CodeGen
00727   // isn't smart enough to ignore the overflow, which leads to much less
00728   // efficient code if the width of the subtraction is wider than the native
00729   // register width.
00730   //
00731   // (It's possible to not widen at all by pulling out factors of 2 before
00732   // the multiplication; for example, K=2 can be calculated as
00733   // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
00734   // extra arithmetic, so it's not an obvious win, and it gets
00735   // much more complicated for K > 3.)
00736 
00737   // Protection from insane SCEVs; this bound is conservative,
00738   // but it probably doesn't matter.
00739   if (K > 1000)
00740     return SE.getCouldNotCompute();
00741 
00742   unsigned W = SE.getTypeSizeInBits(ResultTy);
00743 
00744   // Calculate K! / 2^T and T; we divide out the factors of two before
00745   // multiplying for calculating K! / 2^T to avoid overflow.
00746   // Other overflow doesn't matter because we only care about the bottom
00747   // W bits of the result.
00748   APInt OddFactorial(W, 1);
00749   unsigned T = 1;
00750   for (unsigned i = 3; i <= K; ++i) {
00751     APInt Mult(W, i);
00752     unsigned TwoFactors = Mult.countTrailingZeros();
00753     T += TwoFactors;
00754     Mult = Mult.lshr(TwoFactors);
00755     OddFactorial *= Mult;
00756   }
00757 
00758   // We need at least W + T bits for the multiplication step
00759   unsigned CalculationBits = W + T;
00760 
00761   // Calculate 2^T, at width T+W.
00762   APInt DivFactor = APInt::getOneBitSet(CalculationBits, T);
00763 
00764   // Calculate the multiplicative inverse of K! / 2^T;
00765   // this multiplication factor will perform the exact division by
00766   // K! / 2^T.
00767   APInt Mod = APInt::getSignedMinValue(W+1);
00768   APInt MultiplyFactor = OddFactorial.zext(W+1);
00769   MultiplyFactor = MultiplyFactor.multiplicativeInverse(Mod);
00770   MultiplyFactor = MultiplyFactor.trunc(W);
00771 
00772   // Calculate the product, at width T+W
00773   IntegerType *CalculationTy = IntegerType::get(SE.getContext(),
00774                                                       CalculationBits);
00775   const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy);
00776   for (unsigned i = 1; i != K; ++i) {
00777     const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i));
00778     Dividend = SE.getMulExpr(Dividend,
00779                              SE.getTruncateOrZeroExtend(S, CalculationTy));
00780   }
00781 
00782   // Divide by 2^T
00783   const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor));
00784 
00785   // Truncate the result, and divide by K! / 2^T.
00786 
00787   return SE.getMulExpr(SE.getConstant(MultiplyFactor),
00788                        SE.getTruncateOrZeroExtend(DivResult, ResultTy));
00789 }
00790 
00791 /// evaluateAtIteration - Return the value of this chain of recurrences at
00792 /// the specified iteration number.  We can evaluate this recurrence by
00793 /// multiplying each element in the chain by the binomial coefficient
00794 /// corresponding to it.  In other words, we can evaluate {A,+,B,+,C,+,D} as:
00795 ///
00796 ///   A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
00797 ///
00798 /// where BC(It, k) stands for binomial coefficient.
00799 ///
00800 const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
00801                                                 ScalarEvolution &SE) const {
00802   const SCEV *Result = getStart();
00803   for (unsigned i = 1, e = getNumOperands(); i != e; ++i) {
00804     // The computation is correct in the face of overflow provided that the
00805     // multiplication is performed _after_ the evaluation of the binomial
00806     // coefficient.
00807     const SCEV *Coeff = BinomialCoefficient(It, i, SE, getType());
00808     if (isa<SCEVCouldNotCompute>(Coeff))
00809       return Coeff;
00810 
00811     Result = SE.getAddExpr(Result, SE.getMulExpr(getOperand(i), Coeff));
00812   }
00813   return Result;
00814 }
00815 
00816 //===----------------------------------------------------------------------===//
00817 //                    SCEV Expression folder implementations
00818 //===----------------------------------------------------------------------===//
00819 
00820 const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op,
00821                                              Type *Ty) {
00822   assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
00823          "This is not a truncating conversion!");
00824   assert(isSCEVable(Ty) &&
00825          "This is not a conversion to a SCEVable type!");
00826   Ty = getEffectiveSCEVType(Ty);
00827 
00828   FoldingSetNodeID ID;
00829   ID.AddInteger(scTruncate);
00830   ID.AddPointer(Op);
00831   ID.AddPointer(Ty);
00832   void *IP = nullptr;
00833   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
00834 
00835   // Fold if the operand is constant.
00836   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
00837     return getConstant(
00838       cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(), Ty)));
00839 
00840   // trunc(trunc(x)) --> trunc(x)
00841   if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op))
00842     return getTruncateExpr(ST->getOperand(), Ty);
00843 
00844   // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
00845   if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op))
00846     return getTruncateOrSignExtend(SS->getOperand(), Ty);
00847 
00848   // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
00849   if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
00850     return getTruncateOrZeroExtend(SZ->getOperand(), Ty);
00851 
00852   // trunc(x1+x2+...+xN) --> trunc(x1)+trunc(x2)+...+trunc(xN) if we can
00853   // eliminate all the truncates.
00854   if (const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Op)) {
00855     SmallVector<const SCEV *, 4> Operands;
00856     bool hasTrunc = false;
00857     for (unsigned i = 0, e = SA->getNumOperands(); i != e && !hasTrunc; ++i) {
00858       const SCEV *S = getTruncateExpr(SA->getOperand(i), Ty);
00859       hasTrunc = isa<SCEVTruncateExpr>(S);
00860       Operands.push_back(S);
00861     }
00862     if (!hasTrunc)
00863       return getAddExpr(Operands);
00864     UniqueSCEVs.FindNodeOrInsertPos(ID, IP);  // Mutates IP, returns NULL.
00865   }
00866 
00867   // trunc(x1*x2*...*xN) --> trunc(x1)*trunc(x2)*...*trunc(xN) if we can
00868   // eliminate all the truncates.
00869   if (const SCEVMulExpr *SM = dyn_cast<SCEVMulExpr>(Op)) {
00870     SmallVector<const SCEV *, 4> Operands;
00871     bool hasTrunc = false;
00872     for (unsigned i = 0, e = SM->getNumOperands(); i != e && !hasTrunc; ++i) {
00873       const SCEV *S = getTruncateExpr(SM->getOperand(i), Ty);
00874       hasTrunc = isa<SCEVTruncateExpr>(S);
00875       Operands.push_back(S);
00876     }
00877     if (!hasTrunc)
00878       return getMulExpr(Operands);
00879     UniqueSCEVs.FindNodeOrInsertPos(ID, IP);  // Mutates IP, returns NULL.
00880   }
00881 
00882   // If the input value is a chrec scev, truncate the chrec's operands.
00883   if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) {
00884     SmallVector<const SCEV *, 4> Operands;
00885     for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i)
00886       Operands.push_back(getTruncateExpr(AddRec->getOperand(i), Ty));
00887     return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap);
00888   }
00889 
00890   // The cast wasn't folded; create an explicit cast node. We can reuse
00891   // the existing insert position since if we get here, we won't have
00892   // made any changes which would invalidate it.
00893   SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator),
00894                                                  Op, Ty);
00895   UniqueSCEVs.InsertNode(S, IP);
00896   return S;
00897 }
00898 
00899 const SCEV *ScalarEvolution::getZeroExtendExpr(const SCEV *Op,
00900                                                Type *Ty) {
00901   assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
00902          "This is not an extending conversion!");
00903   assert(isSCEVable(Ty) &&
00904          "This is not a conversion to a SCEVable type!");
00905   Ty = getEffectiveSCEVType(Ty);
00906 
00907   // Fold if the operand is constant.
00908   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
00909     return getConstant(
00910       cast<ConstantInt>(ConstantExpr::getZExt(SC->getValue(), Ty)));
00911 
00912   // zext(zext(x)) --> zext(x)
00913   if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
00914     return getZeroExtendExpr(SZ->getOperand(), Ty);
00915 
00916   // Before doing any expensive analysis, check to see if we've already
00917   // computed a SCEV for this Op and Ty.
00918   FoldingSetNodeID ID;
00919   ID.AddInteger(scZeroExtend);
00920   ID.AddPointer(Op);
00921   ID.AddPointer(Ty);
00922   void *IP = nullptr;
00923   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
00924 
00925   // zext(trunc(x)) --> zext(x) or x or trunc(x)
00926   if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) {
00927     // It's possible the bits taken off by the truncate were all zero bits. If
00928     // so, we should be able to simplify this further.
00929     const SCEV *X = ST->getOperand();
00930     ConstantRange CR = getUnsignedRange(X);
00931     unsigned TruncBits = getTypeSizeInBits(ST->getType());
00932     unsigned NewBits = getTypeSizeInBits(Ty);
00933     if (CR.truncate(TruncBits).zeroExtend(NewBits).contains(
00934             CR.zextOrTrunc(NewBits)))
00935       return getTruncateOrZeroExtend(X, Ty);
00936   }
00937 
00938   // If the input value is a chrec scev, and we can prove that the value
00939   // did not overflow the old, smaller, value, we can zero extend all of the
00940   // operands (often constants).  This allows analysis of something like
00941   // this:  for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
00942   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op))
00943     if (AR->isAffine()) {
00944       const SCEV *Start = AR->getStart();
00945       const SCEV *Step = AR->getStepRecurrence(*this);
00946       unsigned BitWidth = getTypeSizeInBits(AR->getType());
00947       const Loop *L = AR->getLoop();
00948 
00949       // If we have special knowledge that this addrec won't overflow,
00950       // we don't need to do any further analysis.
00951       if (AR->getNoWrapFlags(SCEV::FlagNUW))
00952         return getAddRecExpr(getZeroExtendExpr(Start, Ty),
00953                              getZeroExtendExpr(Step, Ty),
00954                              L, AR->getNoWrapFlags());
00955 
00956       // Check whether the backedge-taken count is SCEVCouldNotCompute.
00957       // Note that this serves two purposes: It filters out loops that are
00958       // simply not analyzable, and it covers the case where this code is
00959       // being called from within backedge-taken count analysis, such that
00960       // attempting to ask for the backedge-taken count would likely result
00961       // in infinite recursion. In the later case, the analysis code will
00962       // cope with a conservative value, and it will take care to purge
00963       // that value once it has finished.
00964       const SCEV *MaxBECount = getMaxBackedgeTakenCount(L);
00965       if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
00966         // Manually compute the final value for AR, checking for
00967         // overflow.
00968 
00969         // Check whether the backedge-taken count can be losslessly casted to
00970         // the addrec's type. The count is always unsigned.
00971         const SCEV *CastedMaxBECount =
00972           getTruncateOrZeroExtend(MaxBECount, Start->getType());
00973         const SCEV *RecastedMaxBECount =
00974           getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType());
00975         if (MaxBECount == RecastedMaxBECount) {
00976           Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
00977           // Check whether Start+Step*MaxBECount has no unsigned overflow.
00978           const SCEV *ZMul = getMulExpr(CastedMaxBECount, Step);
00979           const SCEV *ZAdd = getZeroExtendExpr(getAddExpr(Start, ZMul), WideTy);
00980           const SCEV *WideStart = getZeroExtendExpr(Start, WideTy);
00981           const SCEV *WideMaxBECount =
00982             getZeroExtendExpr(CastedMaxBECount, WideTy);
00983           const SCEV *OperandExtendedAdd =
00984             getAddExpr(WideStart,
00985                        getMulExpr(WideMaxBECount,
00986                                   getZeroExtendExpr(Step, WideTy)));
00987           if (ZAdd == OperandExtendedAdd) {
00988             // Cache knowledge of AR NUW, which is propagated to this AddRec.
00989             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW);
00990             // Return the expression with the addrec on the outside.
00991             return getAddRecExpr(getZeroExtendExpr(Start, Ty),
00992                                  getZeroExtendExpr(Step, Ty),
00993                                  L, AR->getNoWrapFlags());
00994           }
00995           // Similar to above, only this time treat the step value as signed.
00996           // This covers loops that count down.
00997           OperandExtendedAdd =
00998             getAddExpr(WideStart,
00999                        getMulExpr(WideMaxBECount,
01000                                   getSignExtendExpr(Step, WideTy)));
01001           if (ZAdd == OperandExtendedAdd) {
01002             // Cache knowledge of AR NW, which is propagated to this AddRec.
01003             // Negative step causes unsigned wrap, but it still can't self-wrap.
01004             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
01005             // Return the expression with the addrec on the outside.
01006             return getAddRecExpr(getZeroExtendExpr(Start, Ty),
01007                                  getSignExtendExpr(Step, Ty),
01008                                  L, AR->getNoWrapFlags());
01009           }
01010         }
01011 
01012         // If the backedge is guarded by a comparison with the pre-inc value
01013         // the addrec is safe. Also, if the entry is guarded by a comparison
01014         // with the start value and the backedge is guarded by a comparison
01015         // with the post-inc value, the addrec is safe.
01016         if (isKnownPositive(Step)) {
01017           const SCEV *N = getConstant(APInt::getMinValue(BitWidth) -
01018                                       getUnsignedRange(Step).getUnsignedMax());
01019           if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT, AR, N) ||
01020               (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_ULT, Start, N) &&
01021                isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_ULT,
01022                                            AR->getPostIncExpr(*this), N))) {
01023             // Cache knowledge of AR NUW, which is propagated to this AddRec.
01024             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNUW);
01025             // Return the expression with the addrec on the outside.
01026             return getAddRecExpr(getZeroExtendExpr(Start, Ty),
01027                                  getZeroExtendExpr(Step, Ty),
01028                                  L, AR->getNoWrapFlags());
01029           }
01030         } else if (isKnownNegative(Step)) {
01031           const SCEV *N = getConstant(APInt::getMaxValue(BitWidth) -
01032                                       getSignedRange(Step).getSignedMin());
01033           if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, AR, N) ||
01034               (isLoopEntryGuardedByCond(L, ICmpInst::ICMP_UGT, Start, N) &&
01035                isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT,
01036                                            AR->getPostIncExpr(*this), N))) {
01037             // Cache knowledge of AR NW, which is propagated to this AddRec.
01038             // Negative step causes unsigned wrap, but it still can't self-wrap.
01039             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
01040             // Return the expression with the addrec on the outside.
01041             return getAddRecExpr(getZeroExtendExpr(Start, Ty),
01042                                  getSignExtendExpr(Step, Ty),
01043                                  L, AR->getNoWrapFlags());
01044           }
01045         }
01046       }
01047     }
01048 
01049   // The cast wasn't folded; create an explicit cast node.
01050   // Recompute the insert position, as it may have been invalidated.
01051   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01052   SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
01053                                                    Op, Ty);
01054   UniqueSCEVs.InsertNode(S, IP);
01055   return S;
01056 }
01057 
01058 // Get the limit of a recurrence such that incrementing by Step cannot cause
01059 // signed overflow as long as the value of the recurrence within the loop does
01060 // not exceed this limit before incrementing.
01061 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
01062                                            ICmpInst::Predicate *Pred,
01063                                            ScalarEvolution *SE) {
01064   unsigned BitWidth = SE->getTypeSizeInBits(Step->getType());
01065   if (SE->isKnownPositive(Step)) {
01066     *Pred = ICmpInst::ICMP_SLT;
01067     return SE->getConstant(APInt::getSignedMinValue(BitWidth) -
01068                            SE->getSignedRange(Step).getSignedMax());
01069   }
01070   if (SE->isKnownNegative(Step)) {
01071     *Pred = ICmpInst::ICMP_SGT;
01072     return SE->getConstant(APInt::getSignedMaxValue(BitWidth) -
01073                        SE->getSignedRange(Step).getSignedMin());
01074   }
01075   return nullptr;
01076 }
01077 
01078 // The recurrence AR has been shown to have no signed wrap. Typically, if we can
01079 // prove NSW for AR, then we can just as easily prove NSW for its preincrement
01080 // or postincrement sibling. This allows normalizing a sign extended AddRec as
01081 // such: {sext(Step + Start),+,Step} => {(Step + sext(Start),+,Step} As a
01082 // result, the expression "Step + sext(PreIncAR)" is congruent with
01083 // "sext(PostIncAR)"
01084 static const SCEV *getPreStartForSignExtend(const SCEVAddRecExpr *AR,
01085                                             Type *Ty,
01086                                             ScalarEvolution *SE) {
01087   const Loop *L = AR->getLoop();
01088   const SCEV *Start = AR->getStart();
01089   const SCEV *Step = AR->getStepRecurrence(*SE);
01090 
01091   // Check for a simple looking step prior to loop entry.
01092   const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start);
01093   if (!SA)
01094     return nullptr;
01095 
01096   // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
01097   // subtraction is expensive. For this purpose, perform a quick and dirty
01098   // difference, by checking for Step in the operand list.
01099   SmallVector<const SCEV *, 4> DiffOps;
01100   for (SCEVAddExpr::op_iterator I = SA->op_begin(), E = SA->op_end();
01101        I != E; ++I) {
01102     if (*I != Step)
01103       DiffOps.push_back(*I);
01104   }
01105   if (DiffOps.size() == SA->getNumOperands())
01106     return nullptr;
01107 
01108   // This is a postinc AR. Check for overflow on the preinc recurrence using the
01109   // same three conditions that getSignExtendedExpr checks.
01110 
01111   // 1. NSW flags on the step increment.
01112   const SCEV *PreStart = SE->getAddExpr(DiffOps, SA->getNoWrapFlags());
01113   const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
01114     SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagAnyWrap));
01115 
01116   if (PreAR && PreAR->getNoWrapFlags(SCEV::FlagNSW))
01117     return PreStart;
01118 
01119   // 2. Direct overflow check on the step operation's expression.
01120   unsigned BitWidth = SE->getTypeSizeInBits(AR->getType());
01121   Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2);
01122   const SCEV *OperandExtendedStart =
01123     SE->getAddExpr(SE->getSignExtendExpr(PreStart, WideTy),
01124                    SE->getSignExtendExpr(Step, WideTy));
01125   if (SE->getSignExtendExpr(Start, WideTy) == OperandExtendedStart) {
01126     // Cache knowledge of PreAR NSW.
01127     if (PreAR)
01128       const_cast<SCEVAddRecExpr *>(PreAR)->setNoWrapFlags(SCEV::FlagNSW);
01129     // FIXME: this optimization needs a unit test
01130     DEBUG(dbgs() << "SCEV: untested prestart overflow check\n");
01131     return PreStart;
01132   }
01133 
01134   // 3. Loop precondition.
01135   ICmpInst::Predicate Pred;
01136   const SCEV *OverflowLimit = getOverflowLimitForStep(Step, &Pred, SE);
01137 
01138   if (OverflowLimit &&
01139       SE->isLoopEntryGuardedByCond(L, Pred, PreStart, OverflowLimit)) {
01140     return PreStart;
01141   }
01142   return nullptr;
01143 }
01144 
01145 // Get the normalized sign-extended expression for this AddRec's Start.
01146 static const SCEV *getSignExtendAddRecStart(const SCEVAddRecExpr *AR,
01147                                             Type *Ty,
01148                                             ScalarEvolution *SE) {
01149   const SCEV *PreStart = getPreStartForSignExtend(AR, Ty, SE);
01150   if (!PreStart)
01151     return SE->getSignExtendExpr(AR->getStart(), Ty);
01152 
01153   return SE->getAddExpr(SE->getSignExtendExpr(AR->getStepRecurrence(*SE), Ty),
01154                         SE->getSignExtendExpr(PreStart, Ty));
01155 }
01156 
01157 const SCEV *ScalarEvolution::getSignExtendExpr(const SCEV *Op,
01158                                                Type *Ty) {
01159   assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
01160          "This is not an extending conversion!");
01161   assert(isSCEVable(Ty) &&
01162          "This is not a conversion to a SCEVable type!");
01163   Ty = getEffectiveSCEVType(Ty);
01164 
01165   // Fold if the operand is constant.
01166   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
01167     return getConstant(
01168       cast<ConstantInt>(ConstantExpr::getSExt(SC->getValue(), Ty)));
01169 
01170   // sext(sext(x)) --> sext(x)
01171   if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op))
01172     return getSignExtendExpr(SS->getOperand(), Ty);
01173 
01174   // sext(zext(x)) --> zext(x)
01175   if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
01176     return getZeroExtendExpr(SZ->getOperand(), Ty);
01177 
01178   // Before doing any expensive analysis, check to see if we've already
01179   // computed a SCEV for this Op and Ty.
01180   FoldingSetNodeID ID;
01181   ID.AddInteger(scSignExtend);
01182   ID.AddPointer(Op);
01183   ID.AddPointer(Ty);
01184   void *IP = nullptr;
01185   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01186 
01187   // If the input value is provably positive, build a zext instead.
01188   if (isKnownNonNegative(Op))
01189     return getZeroExtendExpr(Op, Ty);
01190 
01191   // sext(trunc(x)) --> sext(x) or x or trunc(x)
01192   if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) {
01193     // It's possible the bits taken off by the truncate were all sign bits. If
01194     // so, we should be able to simplify this further.
01195     const SCEV *X = ST->getOperand();
01196     ConstantRange CR = getSignedRange(X);
01197     unsigned TruncBits = getTypeSizeInBits(ST->getType());
01198     unsigned NewBits = getTypeSizeInBits(Ty);
01199     if (CR.truncate(TruncBits).signExtend(NewBits).contains(
01200             CR.sextOrTrunc(NewBits)))
01201       return getTruncateOrSignExtend(X, Ty);
01202   }
01203 
01204   // If the input value is a chrec scev, and we can prove that the value
01205   // did not overflow the old, smaller, value, we can sign extend all of the
01206   // operands (often constants).  This allows analysis of something like
01207   // this:  for (signed char X = 0; X < 100; ++X) { int Y = X; }
01208   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op))
01209     if (AR->isAffine()) {
01210       const SCEV *Start = AR->getStart();
01211       const SCEV *Step = AR->getStepRecurrence(*this);
01212       unsigned BitWidth = getTypeSizeInBits(AR->getType());
01213       const Loop *L = AR->getLoop();
01214 
01215       // If we have special knowledge that this addrec won't overflow,
01216       // we don't need to do any further analysis.
01217       if (AR->getNoWrapFlags(SCEV::FlagNSW))
01218         return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this),
01219                              getSignExtendExpr(Step, Ty),
01220                              L, SCEV::FlagNSW);
01221 
01222       // Check whether the backedge-taken count is SCEVCouldNotCompute.
01223       // Note that this serves two purposes: It filters out loops that are
01224       // simply not analyzable, and it covers the case where this code is
01225       // being called from within backedge-taken count analysis, such that
01226       // attempting to ask for the backedge-taken count would likely result
01227       // in infinite recursion. In the later case, the analysis code will
01228       // cope with a conservative value, and it will take care to purge
01229       // that value once it has finished.
01230       const SCEV *MaxBECount = getMaxBackedgeTakenCount(L);
01231       if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
01232         // Manually compute the final value for AR, checking for
01233         // overflow.
01234 
01235         // Check whether the backedge-taken count can be losslessly casted to
01236         // the addrec's type. The count is always unsigned.
01237         const SCEV *CastedMaxBECount =
01238           getTruncateOrZeroExtend(MaxBECount, Start->getType());
01239         const SCEV *RecastedMaxBECount =
01240           getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType());
01241         if (MaxBECount == RecastedMaxBECount) {
01242           Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
01243           // Check whether Start+Step*MaxBECount has no signed overflow.
01244           const SCEV *SMul = getMulExpr(CastedMaxBECount, Step);
01245           const SCEV *SAdd = getSignExtendExpr(getAddExpr(Start, SMul), WideTy);
01246           const SCEV *WideStart = getSignExtendExpr(Start, WideTy);
01247           const SCEV *WideMaxBECount =
01248             getZeroExtendExpr(CastedMaxBECount, WideTy);
01249           const SCEV *OperandExtendedAdd =
01250             getAddExpr(WideStart,
01251                        getMulExpr(WideMaxBECount,
01252                                   getSignExtendExpr(Step, WideTy)));
01253           if (SAdd == OperandExtendedAdd) {
01254             // Cache knowledge of AR NSW, which is propagated to this AddRec.
01255             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW);
01256             // Return the expression with the addrec on the outside.
01257             return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this),
01258                                  getSignExtendExpr(Step, Ty),
01259                                  L, AR->getNoWrapFlags());
01260           }
01261           // Similar to above, only this time treat the step value as unsigned.
01262           // This covers loops that count up with an unsigned step.
01263           OperandExtendedAdd =
01264             getAddExpr(WideStart,
01265                        getMulExpr(WideMaxBECount,
01266                                   getZeroExtendExpr(Step, WideTy)));
01267           if (SAdd == OperandExtendedAdd) {
01268             // Cache knowledge of AR NSW, which is propagated to this AddRec.
01269             const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW);
01270             // Return the expression with the addrec on the outside.
01271             return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this),
01272                                  getZeroExtendExpr(Step, Ty),
01273                                  L, AR->getNoWrapFlags());
01274           }
01275         }
01276 
01277         // If the backedge is guarded by a comparison with the pre-inc value
01278         // the addrec is safe. Also, if the entry is guarded by a comparison
01279         // with the start value and the backedge is guarded by a comparison
01280         // with the post-inc value, the addrec is safe.
01281         ICmpInst::Predicate Pred;
01282         const SCEV *OverflowLimit = getOverflowLimitForStep(Step, &Pred, this);
01283         if (OverflowLimit &&
01284             (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) ||
01285              (isLoopEntryGuardedByCond(L, Pred, Start, OverflowLimit) &&
01286               isLoopBackedgeGuardedByCond(L, Pred, AR->getPostIncExpr(*this),
01287                                           OverflowLimit)))) {
01288           // Cache knowledge of AR NSW, then propagate NSW to the wide AddRec.
01289           const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNSW);
01290           return getAddRecExpr(getSignExtendAddRecStart(AR, Ty, this),
01291                                getSignExtendExpr(Step, Ty),
01292                                L, AR->getNoWrapFlags());
01293         }
01294       }
01295     }
01296 
01297   // The cast wasn't folded; create an explicit cast node.
01298   // Recompute the insert position, as it may have been invalidated.
01299   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
01300   SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
01301                                                    Op, Ty);
01302   UniqueSCEVs.InsertNode(S, IP);
01303   return S;
01304 }
01305 
01306 /// getAnyExtendExpr - Return a SCEV for the given operand extended with
01307 /// unspecified bits out to the given type.
01308 ///
01309 const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op,
01310                                               Type *Ty) {
01311   assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
01312          "This is not an extending conversion!");
01313   assert(isSCEVable(Ty) &&
01314          "This is not a conversion to a SCEVable type!");
01315   Ty = getEffectiveSCEVType(Ty);
01316 
01317   // Sign-extend negative constants.
01318   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
01319     if (SC->getValue()->getValue().isNegative())
01320       return getSignExtendExpr(Op, Ty);
01321 
01322   // Peel off a truncate cast.
01323   if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Op)) {
01324     const SCEV *NewOp = T->getOperand();
01325     if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty))
01326       return getAnyExtendExpr(NewOp, Ty);
01327     return getTruncateOrNoop(NewOp, Ty);
01328   }
01329 
01330   // Next try a zext cast. If the cast is folded, use it.
01331   const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
01332   if (!isa<SCEVZeroExtendExpr>(ZExt))
01333     return ZExt;
01334 
01335   // Next try a sext cast. If the cast is folded, use it.
01336   const SCEV *SExt = getSignExtendExpr(Op, Ty);
01337   if (!isa<SCEVSignExtendExpr>(SExt))
01338     return SExt;
01339 
01340   // Force the cast to be folded into the operands of an addrec.
01341   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) {
01342     SmallVector<const SCEV *, 4> Ops;
01343     for (SCEVAddRecExpr::op_iterator I = AR->op_begin(), E = AR->op_end();
01344          I != E; ++I)
01345       Ops.push_back(getAnyExtendExpr(*I, Ty));
01346     return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagNW);
01347   }
01348 
01349   // If the expression is obviously signed, use the sext cast value.
01350   if (isa<SCEVSMaxExpr>(Op))
01351     return SExt;
01352 
01353   // Absent any other information, use the zext cast value.
01354   return ZExt;
01355 }
01356 
01357 /// CollectAddOperandsWithScales - Process the given Ops list, which is
01358 /// a list of operands to be added under the given scale, update the given
01359 /// map. This is a helper function for getAddRecExpr. As an example of
01360 /// what it does, given a sequence of operands that would form an add
01361 /// expression like this:
01362 ///
01363 ///    m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
01364 ///
01365 /// where A and B are constants, update the map with these values:
01366 ///
01367 ///    (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
01368 ///
01369 /// and add 13 + A*B*29 to AccumulatedConstant.
01370 /// This will allow getAddRecExpr to produce this:
01371 ///
01372 ///    13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
01373 ///
01374 /// This form often exposes folding opportunities that are hidden in
01375 /// the original operand list.
01376 ///
01377 /// Return true iff it appears that any interesting folding opportunities
01378 /// may be exposed. This helps getAddRecExpr short-circuit extra work in
01379 /// the common case where no interesting opportunities are present, and
01380 /// is also used as a check to avoid infinite recursion.
01381 ///
01382 static bool
01383 CollectAddOperandsWithScales(DenseMap<const SCEV *, APInt> &M,
01384                              SmallVectorImpl<const SCEV *> &NewOps,
01385                              APInt &AccumulatedConstant,
01386                              const SCEV *const *Ops, size_t NumOperands,
01387                              const APInt &Scale,
01388                              ScalarEvolution &SE) {
01389   bool Interesting = false;
01390 
01391   // Iterate over the add operands. They are sorted, with constants first.
01392   unsigned i = 0;
01393   while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
01394     ++i;
01395     // Pull a buried constant out to the outside.
01396     if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
01397       Interesting = true;
01398     AccumulatedConstant += Scale * C->getValue()->getValue();
01399   }
01400 
01401   // Next comes everything else. We're especially interested in multiplies
01402   // here, but they're in the middle, so just visit the rest with one loop.
01403   for (; i != NumOperands; ++i) {
01404     const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[i]);
01405     if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) {
01406       APInt NewScale =
01407         Scale * cast<SCEVConstant>(Mul->getOperand(0))->getValue()->getValue();
01408       if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) {
01409         // A multiplication of a constant with another add; recurse.
01410         const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1));
01411         Interesting |=
01412           CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
01413                                        Add->op_begin(), Add->getNumOperands(),
01414                                        NewScale, SE);
01415       } else {
01416         // A multiplication of a constant with some other value. Update
01417         // the map.
01418         SmallVector<const SCEV *, 4> MulOps(Mul->op_begin()+1, Mul->op_end());
01419         const SCEV *Key = SE.getMulExpr(MulOps);
01420         std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair =
01421           M.insert(std::make_pair(Key, NewScale));
01422         if (Pair.second) {
01423           NewOps.push_back(Pair.first->first);
01424         } else {
01425           Pair.first->second += NewScale;
01426           // The map already had an entry for this value, which may indicate
01427           // a folding opportunity.
01428           Interesting = true;
01429         }
01430       }
01431     } else {
01432       // An ordinary operand. Update the map.
01433       std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair =
01434         M.insert(std::make_pair(Ops[i], Scale));
01435       if (Pair.second) {
01436         NewOps.push_back(Pair.first->first);
01437       } else {
01438         Pair.first->second += Scale;
01439         // The map already had an entry for this value, which may indicate
01440         // a folding opportunity.
01441         Interesting = true;
01442       }
01443     }
01444   }
01445 
01446   return Interesting;
01447 }
01448 
01449 namespace {
01450   struct APIntCompare {
01451     bool operator()(const APInt &LHS, const APInt &RHS) const {
01452       return LHS.ult(RHS);
01453     }
01454   };
01455 }
01456 
01457 /// getAddExpr - Get a canonical add expression, or something simpler if
01458 /// possible.
01459 const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
01460                                         SCEV::NoWrapFlags Flags) {
01461   assert(!(Flags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
01462          "only nuw or nsw allowed");
01463   assert(!Ops.empty() && "Cannot get empty add!");
01464   if (Ops.size() == 1) return Ops[0];
01465 #ifndef NDEBUG
01466   Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
01467   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
01468     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
01469            "SCEVAddExpr operand types don't match!");
01470 #endif
01471 
01472   // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
01473   // And vice-versa.
01474   int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
01475   SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask);
01476   if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) {
01477     bool All = true;
01478     for (SmallVectorImpl<const SCEV *>::const_iterator I = Ops.begin(),
01479          E = Ops.end(); I != E; ++I)
01480       if (!isKnownNonNegative(*I)) {
01481         All = false;
01482         break;
01483       }
01484     if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask);
01485   }
01486 
01487   // Sort by complexity, this groups all similar expression types together.
01488   GroupByComplexity(Ops, LI);
01489 
01490   // If there are any constants, fold them together.
01491   unsigned Idx = 0;
01492   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
01493     ++Idx;
01494     assert(Idx < Ops.size());
01495     while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
01496       // We found two constants, fold them together!
01497       Ops[0] = getConstant(LHSC->getValue()->getValue() +
01498                            RHSC->getValue()->getValue());
01499       if (Ops.size() == 2) return Ops[0];
01500       Ops.erase(Ops.begin()+1);  // Erase the folded element
01501       LHSC = cast<SCEVConstant>(Ops[0]);
01502     }
01503 
01504     // If we are left with a constant zero being added, strip it off.
01505     if (LHSC->getValue()->isZero()) {
01506       Ops.erase(Ops.begin());
01507       --Idx;
01508     }
01509 
01510     if (Ops.size() == 1) return Ops[0];
01511   }
01512 
01513   // Okay, check to see if the same value occurs in the operand list more than
01514   // once.  If so, merge them together into an multiply expression.  Since we
01515   // sorted the list, these values are required to be adjacent.
01516   Type *Ty = Ops[0]->getType();
01517   bool FoundMatch = false;
01518   for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
01519     if (Ops[i] == Ops[i+1]) {      //  X + Y + Y  -->  X + Y*2
01520       // Scan ahead to count how many equal operands there are.
01521       unsigned Count = 2;
01522       while (i+Count != e && Ops[i+Count] == Ops[i])
01523         ++Count;
01524       // Merge the values into a multiply.
01525       const SCEV *Scale = getConstant(Ty, Count);
01526       const SCEV *Mul = getMulExpr(Scale, Ops[i]);
01527       if (Ops.size() == Count)
01528         return Mul;
01529       Ops[i] = Mul;
01530       Ops.erase(Ops.begin()+i+1, Ops.begin()+i+Count);
01531       --i; e -= Count - 1;
01532       FoundMatch = true;
01533     }
01534   if (FoundMatch)
01535     return getAddExpr(Ops, Flags);
01536 
01537   // Check for truncates. If all the operands are truncated from the same
01538   // type, see if factoring out the truncate would permit the result to be
01539   // folded. eg., trunc(x) + m*trunc(n) --> trunc(x + trunc(m)*n)
01540   // if the contents of the resulting outer trunc fold to something simple.
01541   for (; Idx < Ops.size() && isa<SCEVTruncateExpr>(Ops[Idx]); ++Idx) {
01542     const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Ops[Idx]);
01543     Type *DstType = Trunc->getType();
01544     Type *SrcType = Trunc->getOperand()->getType();
01545     SmallVector<const SCEV *, 8> LargeOps;
01546     bool Ok = true;
01547     // Check all the operands to see if they can be represented in the
01548     // source type of the truncate.
01549     for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
01550       if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Ops[i])) {
01551         if (T->getOperand()->getType() != SrcType) {
01552           Ok = false;
01553           break;
01554         }
01555         LargeOps.push_back(T->getOperand());
01556       } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
01557         LargeOps.push_back(getAnyExtendExpr(C, SrcType));
01558       } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Ops[i])) {
01559         SmallVector<const SCEV *, 8> LargeMulOps;
01560         for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
01561           if (const SCEVTruncateExpr *T =
01562                 dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) {
01563             if (T->getOperand()->getType() != SrcType) {
01564               Ok = false;
01565               break;
01566             }
01567             LargeMulOps.push_back(T->getOperand());
01568           } else if (const SCEVConstant *C =
01569                        dyn_cast<SCEVConstant>(M->getOperand(j))) {
01570             LargeMulOps.push_back(getAnyExtendExpr(C, SrcType));
01571           } else {
01572             Ok = false;
01573             break;
01574           }
01575         }
01576         if (Ok)
01577           LargeOps.push_back(getMulExpr(LargeMulOps));
01578       } else {
01579         Ok = false;
01580         break;
01581       }
01582     }
01583     if (Ok) {
01584       // Evaluate the expression in the larger type.
01585       const SCEV *Fold = getAddExpr(LargeOps, Flags);
01586       // If it folds to something simple, use it. Otherwise, don't.
01587       if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold))
01588         return getTruncateExpr(Fold, DstType);
01589     }
01590   }
01591 
01592   // Skip past any other cast SCEVs.
01593   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
01594     ++Idx;
01595 
01596   // If there are add operands they would be next.
01597   if (Idx < Ops.size()) {
01598     bool DeletedAdd = false;
01599     while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) {
01600       // If we have an add, expand the add operands onto the end of the operands
01601       // list.
01602       Ops.erase(Ops.begin()+Idx);
01603       Ops.append(Add->op_begin(), Add->op_end());
01604       DeletedAdd = true;
01605     }
01606 
01607     // If we deleted at least one add, we added operands to the end of the list,
01608     // and they are not necessarily sorted.  Recurse to resort and resimplify
01609     // any operands we just acquired.
01610     if (DeletedAdd)
01611       return getAddExpr(Ops);
01612   }
01613 
01614   // Skip over the add expression until we get to a multiply.
01615   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
01616     ++Idx;
01617 
01618   // Check to see if there are any folding opportunities present with
01619   // operands multiplied by constant values.
01620   if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) {
01621     uint64_t BitWidth = getTypeSizeInBits(Ty);
01622     DenseMap<const SCEV *, APInt> M;
01623     SmallVector<const SCEV *, 8> NewOps;
01624     APInt AccumulatedConstant(BitWidth, 0);
01625     if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
01626                                      Ops.data(), Ops.size(),
01627                                      APInt(BitWidth, 1), *this)) {
01628       // Some interesting folding opportunity is present, so its worthwhile to
01629       // re-generate the operands list. Group the operands by constant scale,
01630       // to avoid multiplying by the same constant scale multiple times.
01631       std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists;
01632       for (SmallVectorImpl<const SCEV *>::const_iterator I = NewOps.begin(),
01633            E = NewOps.end(); I != E; ++I)
01634         MulOpLists[M.find(*I)->second].push_back(*I);
01635       // Re-generate the operands list.
01636       Ops.clear();
01637       if (AccumulatedConstant != 0)
01638         Ops.push_back(getConstant(AccumulatedConstant));
01639       for (std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare>::iterator
01640            I = MulOpLists.begin(), E = MulOpLists.end(); I != E; ++I)
01641         if (I->first != 0)
01642           Ops.push_back(getMulExpr(getConstant(I->first),
01643                                    getAddExpr(I->second)));
01644       if (Ops.empty())
01645         return getConstant(Ty, 0);
01646       if (Ops.size() == 1)
01647         return Ops[0];
01648       return getAddExpr(Ops);
01649     }
01650   }
01651 
01652   // If we are adding something to a multiply expression, make sure the
01653   // something is not already an operand of the multiply.  If so, merge it into
01654   // the multiply.
01655   for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) {
01656     const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]);
01657     for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
01658       const SCEV *MulOpSCEV = Mul->getOperand(MulOp);
01659       if (isa<SCEVConstant>(MulOpSCEV))
01660         continue;
01661       for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp)
01662         if (MulOpSCEV == Ops[AddOp]) {
01663           // Fold W + X + (X * Y * Z)  -->  W + (X * ((Y*Z)+1))
01664           const SCEV *InnerMul = Mul->getOperand(MulOp == 0);
01665           if (Mul->getNumOperands() != 2) {
01666             // If the multiply has more than two operands, we must get the
01667             // Y*Z term.
01668             SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(),
01669                                                 Mul->op_begin()+MulOp);
01670             MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end());
01671             InnerMul = getMulExpr(MulOps);
01672           }
01673           const SCEV *One = getConstant(Ty, 1);
01674           const SCEV *AddOne = getAddExpr(One, InnerMul);
01675           const SCEV *OuterMul = getMulExpr(AddOne, MulOpSCEV);
01676           if (Ops.size() == 2) return OuterMul;
01677           if (AddOp < Idx) {
01678             Ops.erase(Ops.begin()+AddOp);
01679             Ops.erase(Ops.begin()+Idx-1);
01680           } else {
01681             Ops.erase(Ops.begin()+Idx);
01682             Ops.erase(Ops.begin()+AddOp-1);
01683           }
01684           Ops.push_back(OuterMul);
01685           return getAddExpr(Ops);
01686         }
01687 
01688       // Check this multiply against other multiplies being added together.
01689       for (unsigned OtherMulIdx = Idx+1;
01690            OtherMulIdx < Ops.size() && isa<SCEVMulExpr>(Ops[OtherMulIdx]);
01691            ++OtherMulIdx) {
01692         const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[OtherMulIdx]);
01693         // If MulOp occurs in OtherMul, we can fold the two multiplies
01694         // together.
01695         for (unsigned OMulOp = 0, e = OtherMul->getNumOperands();
01696              OMulOp != e; ++OMulOp)
01697           if (OtherMul->getOperand(OMulOp) == MulOpSCEV) {
01698             // Fold X + (A*B*C) + (A*D*E) --> X + (A*(B*C+D*E))
01699             const SCEV *InnerMul1 = Mul->getOperand(MulOp == 0);
01700             if (Mul->getNumOperands() != 2) {
01701               SmallVector<const SCEV *, 4> MulOps(Mul->op_begin(),
01702                                                   Mul->op_begin()+MulOp);
01703               MulOps.append(Mul->op_begin()+MulOp+1, Mul->op_end());
01704               InnerMul1 = getMulExpr(MulOps);
01705             }
01706             const SCEV *InnerMul2 = OtherMul->getOperand(OMulOp == 0);
01707             if (OtherMul->getNumOperands() != 2) {
01708               SmallVector<const SCEV *, 4> MulOps(OtherMul->op_begin(),
01709                                                   OtherMul->op_begin()+OMulOp);
01710               MulOps.append(OtherMul->op_begin()+OMulOp+1, OtherMul->op_end());
01711               InnerMul2 = getMulExpr(MulOps);
01712             }
01713             const SCEV *InnerMulSum = getAddExpr(InnerMul1,InnerMul2);
01714             const SCEV *OuterMul = getMulExpr(MulOpSCEV, InnerMulSum);
01715             if (Ops.size() == 2) return OuterMul;
01716             Ops.erase(Ops.begin()+Idx);
01717             Ops.erase(Ops.begin()+OtherMulIdx-1);
01718             Ops.push_back(OuterMul);
01719             return getAddExpr(Ops);
01720           }
01721       }
01722     }
01723   }
01724 
01725   // If there are any add recurrences in the operands list, see if any other
01726   // added values are loop invariant.  If so, we can fold them into the
01727   // recurrence.
01728   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
01729     ++Idx;
01730 
01731   // Scan over all recurrences, trying to fold loop invariants into them.
01732   for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
01733     // Scan all of the other operands to this add and add them to the vector if
01734     // they are loop invariant w.r.t. the recurrence.
01735     SmallVector<const SCEV *, 8> LIOps;
01736     const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
01737     const Loop *AddRecLoop = AddRec->getLoop();
01738     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
01739       if (isLoopInvariant(Ops[i], AddRecLoop)) {
01740         LIOps.push_back(Ops[i]);
01741         Ops.erase(Ops.begin()+i);
01742         --i; --e;
01743       }
01744 
01745     // If we found some loop invariants, fold them into the recurrence.
01746     if (!LIOps.empty()) {
01747       //  NLI + LI + {Start,+,Step}  -->  NLI + {LI+Start,+,Step}
01748       LIOps.push_back(AddRec->getStart());
01749 
01750       SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(),
01751                                              AddRec->op_end());
01752       AddRecOps[0] = getAddExpr(LIOps);
01753 
01754       // Build the new addrec. Propagate the NUW and NSW flags if both the
01755       // outer add and the inner addrec are guaranteed to have no overflow.
01756       // Always propagate NW.
01757       Flags = AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW));
01758       const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags);
01759 
01760       // If all of the other operands were loop invariant, we are done.
01761       if (Ops.size() == 1) return NewRec;
01762 
01763       // Otherwise, add the folded AddRec by the non-invariant parts.
01764       for (unsigned i = 0;; ++i)
01765         if (Ops[i] == AddRec) {
01766           Ops[i] = NewRec;
01767           break;
01768         }
01769       return getAddExpr(Ops);
01770     }
01771 
01772     // Okay, if there weren't any loop invariants to be folded, check to see if
01773     // there are multiple AddRec's with the same loop induction variable being
01774     // added together.  If so, we can fold them.
01775     for (unsigned OtherIdx = Idx+1;
01776          OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
01777          ++OtherIdx)
01778       if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) {
01779         // Other + {A,+,B}<L> + {C,+,D}<L>  -->  Other + {A+C,+,B+D}<L>
01780         SmallVector<const SCEV *, 4> AddRecOps(AddRec->op_begin(),
01781                                                AddRec->op_end());
01782         for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
01783              ++OtherIdx)
01784           if (const SCEVAddRecExpr *OtherAddRec =
01785                 dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]))
01786             if (OtherAddRec->getLoop() == AddRecLoop) {
01787               for (unsigned i = 0, e = OtherAddRec->getNumOperands();
01788                    i != e; ++i) {
01789                 if (i >= AddRecOps.size()) {
01790                   AddRecOps.append(OtherAddRec->op_begin()+i,
01791                                    OtherAddRec->op_end());
01792                   break;
01793                 }
01794                 AddRecOps[i] = getAddExpr(AddRecOps[i],
01795                                           OtherAddRec->getOperand(i));
01796               }
01797               Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
01798             }
01799         // Step size has changed, so we cannot guarantee no self-wraparound.
01800         Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap);
01801         return getAddExpr(Ops);
01802       }
01803 
01804     // Otherwise couldn't fold anything into this recurrence.  Move onto the
01805     // next one.
01806   }
01807 
01808   // Okay, it looks like we really DO need an add expr.  Check to see if we
01809   // already have one, otherwise create a new one.
01810   FoldingSetNodeID ID;
01811   ID.AddInteger(scAddExpr);
01812   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
01813     ID.AddPointer(Ops[i]);
01814   void *IP = nullptr;
01815   SCEVAddExpr *S =
01816     static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
01817   if (!S) {
01818     const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
01819     std::uninitialized_copy(Ops.begin(), Ops.end(), O);
01820     S = new (SCEVAllocator) SCEVAddExpr(ID.Intern(SCEVAllocator),
01821                                         O, Ops.size());
01822     UniqueSCEVs.InsertNode(S, IP);
01823   }
01824   S->setNoWrapFlags(Flags);
01825   return S;
01826 }
01827 
01828 static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
01829   uint64_t k = i*j;
01830   if (j > 1 && k / j != i) Overflow = true;
01831   return k;
01832 }
01833 
01834 /// Compute the result of "n choose k", the binomial coefficient.  If an
01835 /// intermediate computation overflows, Overflow will be set and the return will
01836 /// be garbage. Overflow is not cleared on absence of overflow.
01837 static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
01838   // We use the multiplicative formula:
01839   //     n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
01840   // At each iteration, we take the n-th term of the numeral and divide by the
01841   // (k-n)th term of the denominator.  This division will always produce an
01842   // integral result, and helps reduce the chance of overflow in the
01843   // intermediate computations. However, we can still overflow even when the
01844   // final result would fit.
01845 
01846   if (n == 0 || n == k) return 1;
01847   if (k > n) return 0;
01848 
01849   if (k > n/2)
01850     k = n-k;
01851 
01852   uint64_t r = 1;
01853   for (uint64_t i = 1; i <= k; ++i) {
01854     r = umul_ov(r, n-(i-1), Overflow);
01855     r /= i;
01856   }
01857   return r;
01858 }
01859 
01860 /// getMulExpr - Get a canonical multiply expression, or something simpler if
01861 /// possible.
01862 const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
01863                                         SCEV::NoWrapFlags Flags) {
01864   assert(Flags == maskFlags(Flags, SCEV::FlagNUW | SCEV::FlagNSW) &&
01865          "only nuw or nsw allowed");
01866   assert(!Ops.empty() && "Cannot get empty mul!");
01867   if (Ops.size() == 1) return Ops[0];
01868 #ifndef NDEBUG
01869   Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
01870   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
01871     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
01872            "SCEVMulExpr operand types don't match!");
01873 #endif
01874 
01875   // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
01876   // And vice-versa.
01877   int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
01878   SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask);
01879   if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) {
01880     bool All = true;
01881     for (SmallVectorImpl<const SCEV *>::const_iterator I = Ops.begin(),
01882          E = Ops.end(); I != E; ++I)
01883       if (!isKnownNonNegative(*I)) {
01884         All = false;
01885         break;
01886       }
01887     if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask);
01888   }
01889 
01890   // Sort by complexity, this groups all similar expression types together.
01891   GroupByComplexity(Ops, LI);
01892 
01893   // If there are any constants, fold them together.
01894   unsigned Idx = 0;
01895   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
01896 
01897     // C1*(C2+V) -> C1*C2 + C1*V
01898     if (Ops.size() == 2)
01899       if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1]))
01900         if (Add->getNumOperands() == 2 &&
01901             isa<SCEVConstant>(Add->getOperand(0)))
01902           return getAddExpr(getMulExpr(LHSC, Add->getOperand(0)),
01903                             getMulExpr(LHSC, Add->getOperand(1)));
01904 
01905     ++Idx;
01906     while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
01907       // We found two constants, fold them together!
01908       ConstantInt *Fold = ConstantInt::get(getContext(),
01909                                            LHSC->getValue()->getValue() *
01910                                            RHSC->getValue()->getValue());
01911       Ops[0] = getConstant(Fold);
01912       Ops.erase(Ops.begin()+1);  // Erase the folded element
01913       if (Ops.size() == 1) return Ops[0];
01914       LHSC = cast<SCEVConstant>(Ops[0]);
01915     }
01916 
01917     // If we are left with a constant one being multiplied, strip it off.
01918     if (cast<SCEVConstant>(Ops[0])->getValue()->equalsInt(1)) {
01919       Ops.erase(Ops.begin());
01920       --Idx;
01921     } else if (cast<SCEVConstant>(Ops[0])->getValue()->isZero()) {
01922       // If we have a multiply of zero, it will always be zero.
01923       return Ops[0];
01924     } else if (Ops[0]->isAllOnesValue()) {
01925       // If we have a mul by -1 of an add, try distributing the -1 among the
01926       // add operands.
01927       if (Ops.size() == 2) {
01928         if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) {
01929           SmallVector<const SCEV *, 4> NewOps;
01930           bool AnyFolded = false;
01931           for (SCEVAddRecExpr::op_iterator I = Add->op_begin(),
01932                  E = Add->op_end(); I != E; ++I) {
01933             const SCEV *Mul = getMulExpr(Ops[0], *I);
01934             if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true;
01935             NewOps.push_back(Mul);
01936           }
01937           if (AnyFolded)
01938             return getAddExpr(NewOps);
01939         }
01940         else if (const SCEVAddRecExpr *
01941                  AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) {
01942           // Negation preserves a recurrence's no self-wrap property.
01943           SmallVector<const SCEV *, 4> Operands;
01944           for (SCEVAddRecExpr::op_iterator I = AddRec->op_begin(),
01945                  E = AddRec->op_end(); I != E; ++I) {
01946             Operands.push_back(getMulExpr(Ops[0], *I));
01947           }
01948           return getAddRecExpr(Operands, AddRec->getLoop(),
01949                                AddRec->getNoWrapFlags(SCEV::FlagNW));
01950         }
01951       }
01952     }
01953 
01954     if (Ops.size() == 1)
01955       return Ops[0];
01956   }
01957 
01958   // Skip over the add expression until we get to a multiply.
01959   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
01960     ++Idx;
01961 
01962   // If there are mul operands inline them all into this expression.
01963   if (Idx < Ops.size()) {
01964     bool DeletedMul = false;
01965     while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
01966       // If we have an mul, expand the mul operands onto the end of the operands
01967       // list.
01968       Ops.erase(Ops.begin()+Idx);
01969       Ops.append(Mul->op_begin(), Mul->op_end());
01970       DeletedMul = true;
01971     }
01972 
01973     // If we deleted at least one mul, we added operands to the end of the list,
01974     // and they are not necessarily sorted.  Recurse to resort and resimplify
01975     // any operands we just acquired.
01976     if (DeletedMul)
01977       return getMulExpr(Ops);
01978   }
01979 
01980   // If there are any add recurrences in the operands list, see if any other
01981   // added values are loop invariant.  If so, we can fold them into the
01982   // recurrence.
01983   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
01984     ++Idx;
01985 
01986   // Scan over all recurrences, trying to fold loop invariants into them.
01987   for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
01988     // Scan all of the other operands to this mul and add them to the vector if
01989     // they are loop invariant w.r.t. the recurrence.
01990     SmallVector<const SCEV *, 8> LIOps;
01991     const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
01992     const Loop *AddRecLoop = AddRec->getLoop();
01993     for (unsigned i = 0, e = Ops.size(); i != e; ++i)
01994       if (isLoopInvariant(Ops[i], AddRecLoop)) {
01995         LIOps.push_back(Ops[i]);
01996         Ops.erase(Ops.begin()+i);
01997         --i; --e;
01998       }
01999 
02000     // If we found some loop invariants, fold them into the recurrence.
02001     if (!LIOps.empty()) {
02002       //  NLI * LI * {Start,+,Step}  -->  NLI * {LI*Start,+,LI*Step}
02003       SmallVector<const SCEV *, 4> NewOps;
02004       NewOps.reserve(AddRec->getNumOperands());
02005       const SCEV *Scale = getMulExpr(LIOps);
02006       for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i)
02007         NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i)));
02008 
02009       // Build the new addrec. Propagate the NUW and NSW flags if both the
02010       // outer mul and the inner addrec are guaranteed to have no overflow.
02011       //
02012       // No self-wrap cannot be guaranteed after changing the step size, but
02013       // will be inferred if either NUW or NSW is true.
02014       Flags = AddRec->getNoWrapFlags(clearFlags(Flags, SCEV::FlagNW));
02015       const SCEV *NewRec = getAddRecExpr(NewOps, AddRecLoop, Flags);
02016 
02017       // If all of the other operands were loop invariant, we are done.
02018       if (Ops.size() == 1) return NewRec;
02019 
02020       // Otherwise, multiply the folded AddRec by the non-invariant parts.
02021       for (unsigned i = 0;; ++i)
02022         if (Ops[i] == AddRec) {
02023           Ops[i] = NewRec;
02024           break;
02025         }
02026       return getMulExpr(Ops);
02027     }
02028 
02029     // Okay, if there weren't any loop invariants to be folded, check to see if
02030     // there are multiple AddRec's with the same loop induction variable being
02031     // multiplied together.  If so, we can fold them.
02032     for (unsigned OtherIdx = Idx+1;
02033          OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
02034          ++OtherIdx) {
02035       if (AddRecLoop != cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop())
02036         continue;
02037 
02038       // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
02039       // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
02040       //       choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
02041       //   ]]],+,...up to x=2n}.
02042       // Note that the arguments to choose() are always integers with values
02043       // known at compile time, never SCEV objects.
02044       //
02045       // The implementation avoids pointless extra computations when the two
02046       // addrec's are of different length (mathematically, it's equivalent to
02047       // an infinite stream of zeros on the right).
02048       bool OpsModified = false;
02049       for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
02050            ++OtherIdx) {
02051         const SCEVAddRecExpr *OtherAddRec =
02052           dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]);
02053         if (!OtherAddRec || OtherAddRec->getLoop() != AddRecLoop)
02054           continue;
02055 
02056         bool Overflow = false;
02057         Type *Ty = AddRec->getType();
02058         bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
02059         SmallVector<const SCEV*, 7> AddRecOps;
02060         for (int x = 0, xe = AddRec->getNumOperands() +
02061                OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
02062           const SCEV *Term = getConstant(Ty, 0);
02063           for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
02064             uint64_t Coeff1 = Choose(x, 2*x - y, Overflow);
02065             for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1),
02066                    ze = std::min(x+1, (int)OtherAddRec->getNumOperands());
02067                  z < ze && !Overflow; ++z) {
02068               uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow);
02069               uint64_t Coeff;
02070               if (LargerThan64Bits)
02071                 Coeff = umul_ov(Coeff1, Coeff2, Overflow);
02072               else
02073                 Coeff = Coeff1*Coeff2;
02074               const SCEV *CoeffTerm = getConstant(Ty, Coeff);
02075               const SCEV *Term1 = AddRec->getOperand(y-z);
02076               const SCEV *Term2 = OtherAddRec->getOperand(z);
02077               Term = getAddExpr(Term, getMulExpr(CoeffTerm, Term1,Term2));
02078             }
02079           }
02080           AddRecOps.push_back(Term);
02081         }
02082         if (!Overflow) {
02083           const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRec->getLoop(),
02084                                                 SCEV::FlagAnyWrap);
02085           if (Ops.size() == 2) return NewAddRec;
02086           Ops[Idx] = NewAddRec;
02087           Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
02088           OpsModified = true;
02089           AddRec = dyn_cast<SCEVAddRecExpr>(NewAddRec);
02090           if (!AddRec)
02091             break;
02092         }
02093       }
02094       if (OpsModified)
02095         return getMulExpr(Ops);
02096     }
02097 
02098     // Otherwise couldn't fold anything into this recurrence.  Move onto the
02099     // next one.
02100   }
02101 
02102   // Okay, it looks like we really DO need an mul expr.  Check to see if we
02103   // already have one, otherwise create a new one.
02104   FoldingSetNodeID ID;
02105   ID.AddInteger(scMulExpr);
02106   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02107     ID.AddPointer(Ops[i]);
02108   void *IP = nullptr;
02109   SCEVMulExpr *S =
02110     static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
02111   if (!S) {
02112     const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
02113     std::uninitialized_copy(Ops.begin(), Ops.end(), O);
02114     S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator),
02115                                         O, Ops.size());
02116     UniqueSCEVs.InsertNode(S, IP);
02117   }
02118   S->setNoWrapFlags(Flags);
02119   return S;
02120 }
02121 
02122 /// getUDivExpr - Get a canonical unsigned division expression, or something
02123 /// simpler if possible.
02124 const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS,
02125                                          const SCEV *RHS) {
02126   assert(getEffectiveSCEVType(LHS->getType()) ==
02127          getEffectiveSCEVType(RHS->getType()) &&
02128          "SCEVUDivExpr operand types don't match!");
02129 
02130   if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
02131     if (RHSC->getValue()->equalsInt(1))
02132       return LHS;                               // X udiv 1 --> x
02133     // If the denominator is zero, the result of the udiv is undefined. Don't
02134     // try to analyze it, because the resolution chosen here may differ from
02135     // the resolution chosen in other parts of the compiler.
02136     if (!RHSC->getValue()->isZero()) {
02137       // Determine if the division can be folded into the operands of
02138       // its operands.
02139       // TODO: Generalize this to non-constants by using known-bits information.
02140       Type *Ty = LHS->getType();
02141       unsigned LZ = RHSC->getValue()->getValue().countLeadingZeros();
02142       unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
02143       // For non-power-of-two values, effectively round the value up to the
02144       // nearest power of two.
02145       if (!RHSC->getValue()->getValue().isPowerOf2())
02146         ++MaxShiftAmt;
02147       IntegerType *ExtTy =
02148         IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt);
02149       if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
02150         if (const SCEVConstant *Step =
02151             dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) {
02152           // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
02153           const APInt &StepInt = Step->getValue()->getValue();
02154           const APInt &DivInt = RHSC->getValue()->getValue();
02155           if (!StepInt.urem(DivInt) &&
02156               getZeroExtendExpr(AR, ExtTy) ==
02157               getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
02158                             getZeroExtendExpr(Step, ExtTy),
02159                             AR->getLoop(), SCEV::FlagAnyWrap)) {
02160             SmallVector<const SCEV *, 4> Operands;
02161             for (unsigned i = 0, e = AR->getNumOperands(); i != e; ++i)
02162               Operands.push_back(getUDivExpr(AR->getOperand(i), RHS));
02163             return getAddRecExpr(Operands, AR->getLoop(),
02164                                  SCEV::FlagNW);
02165           }
02166           /// Get a canonical UDivExpr for a recurrence.
02167           /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
02168           // We can currently only fold X%N if X is constant.
02169           const SCEVConstant *StartC = dyn_cast<SCEVConstant>(AR->getStart());
02170           if (StartC && !DivInt.urem(StepInt) &&
02171               getZeroExtendExpr(AR, ExtTy) ==
02172               getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
02173                             getZeroExtendExpr(Step, ExtTy),
02174                             AR->getLoop(), SCEV::FlagAnyWrap)) {
02175             const APInt &StartInt = StartC->getValue()->getValue();
02176             const APInt &StartRem = StartInt.urem(StepInt);
02177             if (StartRem != 0)
02178               LHS = getAddRecExpr(getConstant(StartInt - StartRem), Step,
02179                                   AR->getLoop(), SCEV::FlagNW);
02180           }
02181         }
02182       // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
02183       if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) {
02184         SmallVector<const SCEV *, 4> Operands;
02185         for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i)
02186           Operands.push_back(getZeroExtendExpr(M->getOperand(i), ExtTy));
02187         if (getZeroExtendExpr(M, ExtTy) == getMulExpr(Operands))
02188           // Find an operand that's safely divisible.
02189           for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
02190             const SCEV *Op = M->getOperand(i);
02191             const SCEV *Div = getUDivExpr(Op, RHSC);
02192             if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) {
02193               Operands = SmallVector<const SCEV *, 4>(M->op_begin(),
02194                                                       M->op_end());
02195               Operands[i] = Div;
02196               return getMulExpr(Operands);
02197             }
02198           }
02199       }
02200       // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded.
02201       if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) {
02202         SmallVector<const SCEV *, 4> Operands;
02203         for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i)
02204           Operands.push_back(getZeroExtendExpr(A->getOperand(i), ExtTy));
02205         if (getZeroExtendExpr(A, ExtTy) == getAddExpr(Operands)) {
02206           Operands.clear();
02207           for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
02208             const SCEV *Op = getUDivExpr(A->getOperand(i), RHS);
02209             if (isa<SCEVUDivExpr>(Op) ||
02210                 getMulExpr(Op, RHS) != A->getOperand(i))
02211               break;
02212             Operands.push_back(Op);
02213           }
02214           if (Operands.size() == A->getNumOperands())
02215             return getAddExpr(Operands);
02216         }
02217       }
02218 
02219       // Fold if both operands are constant.
02220       if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
02221         Constant *LHSCV = LHSC->getValue();
02222         Constant *RHSCV = RHSC->getValue();
02223         return getConstant(cast<ConstantInt>(ConstantExpr::getUDiv(LHSCV,
02224                                                                    RHSCV)));
02225       }
02226     }
02227   }
02228 
02229   FoldingSetNodeID ID;
02230   ID.AddInteger(scUDivExpr);
02231   ID.AddPointer(LHS);
02232   ID.AddPointer(RHS);
02233   void *IP = nullptr;
02234   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
02235   SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator),
02236                                              LHS, RHS);
02237   UniqueSCEVs.InsertNode(S, IP);
02238   return S;
02239 }
02240 
02241 static const APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) {
02242   APInt A = C1->getValue()->getValue().abs();
02243   APInt B = C2->getValue()->getValue().abs();
02244   uint32_t ABW = A.getBitWidth();
02245   uint32_t BBW = B.getBitWidth();
02246 
02247   if (ABW > BBW)
02248     B = B.zext(ABW);
02249   else if (ABW < BBW)
02250     A = A.zext(BBW);
02251 
02252   return APIntOps::GreatestCommonDivisor(A, B);
02253 }
02254 
02255 /// getUDivExactExpr - Get a canonical unsigned division expression, or
02256 /// something simpler if possible. There is no representation for an exact udiv
02257 /// in SCEV IR, but we can attempt to remove factors from the LHS and RHS.
02258 /// We can't do this when it's not exact because the udiv may be clearing bits.
02259 const SCEV *ScalarEvolution::getUDivExactExpr(const SCEV *LHS,
02260                                               const SCEV *RHS) {
02261   // TODO: we could try to find factors in all sorts of things, but for now we
02262   // just deal with u/exact (multiply, constant). See SCEVDivision towards the
02263   // end of this file for inspiration.
02264 
02265   const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS);
02266   if (!Mul)
02267     return getUDivExpr(LHS, RHS);
02268 
02269   if (const SCEVConstant *RHSCst = dyn_cast<SCEVConstant>(RHS)) {
02270     // If the mulexpr multiplies by a constant, then that constant must be the
02271     // first element of the mulexpr.
02272     if (const SCEVConstant *LHSCst =
02273             dyn_cast<SCEVConstant>(Mul->getOperand(0))) {
02274       if (LHSCst == RHSCst) {
02275         SmallVector<const SCEV *, 2> Operands;
02276         Operands.append(Mul->op_begin() + 1, Mul->op_end());
02277         return getMulExpr(Operands);
02278       }
02279 
02280       // We can't just assume that LHSCst divides RHSCst cleanly, it could be
02281       // that there's a factor provided by one of the other terms. We need to
02282       // check.
02283       APInt Factor = gcd(LHSCst, RHSCst);
02284       if (!Factor.isIntN(1)) {
02285         LHSCst = cast<SCEVConstant>(
02286             getConstant(LHSCst->getValue()->getValue().udiv(Factor)));
02287         RHSCst = cast<SCEVConstant>(
02288             getConstant(RHSCst->getValue()->getValue().udiv(Factor)));
02289         SmallVector<const SCEV *, 2> Operands;
02290         Operands.push_back(LHSCst);
02291         Operands.append(Mul->op_begin() + 1, Mul->op_end());
02292         LHS = getMulExpr(Operands);
02293         RHS = RHSCst;
02294         Mul = dyn_cast<SCEVMulExpr>(LHS);
02295         if (!Mul)
02296           return getUDivExactExpr(LHS, RHS);
02297       }
02298     }
02299   }
02300 
02301   for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
02302     if (Mul->getOperand(i) == RHS) {
02303       SmallVector<const SCEV *, 2> Operands;
02304       Operands.append(Mul->op_begin(), Mul->op_begin() + i);
02305       Operands.append(Mul->op_begin() + i + 1, Mul->op_end());
02306       return getMulExpr(Operands);
02307     }
02308   }
02309 
02310   return getUDivExpr(LHS, RHS);
02311 }
02312 
02313 /// getAddRecExpr - Get an add recurrence expression for the specified loop.
02314 /// Simplify the expression as much as possible.
02315 const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, const SCEV *Step,
02316                                            const Loop *L,
02317                                            SCEV::NoWrapFlags Flags) {
02318   SmallVector<const SCEV *, 4> Operands;
02319   Operands.push_back(Start);
02320   if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step))
02321     if (StepChrec->getLoop() == L) {
02322       Operands.append(StepChrec->op_begin(), StepChrec->op_end());
02323       return getAddRecExpr(Operands, L, maskFlags(Flags, SCEV::FlagNW));
02324     }
02325 
02326   Operands.push_back(Step);
02327   return getAddRecExpr(Operands, L, Flags);
02328 }
02329 
02330 /// getAddRecExpr - Get an add recurrence expression for the specified loop.
02331 /// Simplify the expression as much as possible.
02332 const SCEV *
02333 ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands,
02334                                const Loop *L, SCEV::NoWrapFlags Flags) {
02335   if (Operands.size() == 1) return Operands[0];
02336 #ifndef NDEBUG
02337   Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
02338   for (unsigned i = 1, e = Operands.size(); i != e; ++i)
02339     assert(getEffectiveSCEVType(Operands[i]->getType()) == ETy &&
02340            "SCEVAddRecExpr operand types don't match!");
02341   for (unsigned i = 0, e = Operands.size(); i != e; ++i)
02342     assert(isLoopInvariant(Operands[i], L) &&
02343            "SCEVAddRecExpr operand is not loop-invariant!");
02344 #endif
02345 
02346   if (Operands.back()->isZero()) {
02347     Operands.pop_back();
02348     return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); // {X,+,0}  -->  X
02349   }
02350 
02351   // It's tempting to want to call getMaxBackedgeTakenCount count here and
02352   // use that information to infer NUW and NSW flags. However, computing a
02353   // BE count requires calling getAddRecExpr, so we may not yet have a
02354   // meaningful BE count at this point (and if we don't, we'd be stuck
02355   // with a SCEVCouldNotCompute as the cached BE count).
02356 
02357   // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
02358   // And vice-versa.
02359   int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
02360   SCEV::NoWrapFlags SignOrUnsignWrap = maskFlags(Flags, SignOrUnsignMask);
02361   if (SignOrUnsignWrap && (SignOrUnsignWrap != SignOrUnsignMask)) {
02362     bool All = true;
02363     for (SmallVectorImpl<const SCEV *>::const_iterator I = Operands.begin(),
02364          E = Operands.end(); I != E; ++I)
02365       if (!isKnownNonNegative(*I)) {
02366         All = false;
02367         break;
02368       }
02369     if (All) Flags = setFlags(Flags, (SCEV::NoWrapFlags)SignOrUnsignMask);
02370   }
02371 
02372   // Canonicalize nested AddRecs in by nesting them in order of loop depth.
02373   if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) {
02374     const Loop *NestedLoop = NestedAR->getLoop();
02375     if (L->contains(NestedLoop) ?
02376         (L->getLoopDepth() < NestedLoop->getLoopDepth()) :
02377         (!NestedLoop->contains(L) &&
02378          DT->dominates(L->getHeader(), NestedLoop->getHeader()))) {
02379       SmallVector<const SCEV *, 4> NestedOperands(NestedAR->op_begin(),
02380                                                   NestedAR->op_end());
02381       Operands[0] = NestedAR->getStart();
02382       // AddRecs require their operands be loop-invariant with respect to their
02383       // loops. Don't perform this transformation if it would break this
02384       // requirement.
02385       bool AllInvariant = true;
02386       for (unsigned i = 0, e = Operands.size(); i != e; ++i)
02387         if (!isLoopInvariant(Operands[i], L)) {
02388           AllInvariant = false;
02389           break;
02390         }
02391       if (AllInvariant) {
02392         // Create a recurrence for the outer loop with the same step size.
02393         //
02394         // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the
02395         // inner recurrence has the same property.
02396         SCEV::NoWrapFlags OuterFlags =
02397           maskFlags(Flags, SCEV::FlagNW | NestedAR->getNoWrapFlags());
02398 
02399         NestedOperands[0] = getAddRecExpr(Operands, L, OuterFlags);
02400         AllInvariant = true;
02401         for (unsigned i = 0, e = NestedOperands.size(); i != e; ++i)
02402           if (!isLoopInvariant(NestedOperands[i], NestedLoop)) {
02403             AllInvariant = false;
02404             break;
02405           }
02406         if (AllInvariant) {
02407           // Ok, both add recurrences are valid after the transformation.
02408           //
02409           // The inner recurrence keeps its NW flag but only keeps NUW/NSW if
02410           // the outer recurrence has the same property.
02411           SCEV::NoWrapFlags InnerFlags =
02412             maskFlags(NestedAR->getNoWrapFlags(), SCEV::FlagNW | Flags);
02413           return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags);
02414         }
02415       }
02416       // Reset Operands to its original state.
02417       Operands[0] = NestedAR;
02418     }
02419   }
02420 
02421   // Okay, it looks like we really DO need an addrec expr.  Check to see if we
02422   // already have one, otherwise create a new one.
02423   FoldingSetNodeID ID;
02424   ID.AddInteger(scAddRecExpr);
02425   for (unsigned i = 0, e = Operands.size(); i != e; ++i)
02426     ID.AddPointer(Operands[i]);
02427   ID.AddPointer(L);
02428   void *IP = nullptr;
02429   SCEVAddRecExpr *S =
02430     static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, IP));
02431   if (!S) {
02432     const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Operands.size());
02433     std::uninitialized_copy(Operands.begin(), Operands.end(), O);
02434     S = new (SCEVAllocator) SCEVAddRecExpr(ID.Intern(SCEVAllocator),
02435                                            O, Operands.size(), L);
02436     UniqueSCEVs.InsertNode(S, IP);
02437   }
02438   S->setNoWrapFlags(Flags);
02439   return S;
02440 }
02441 
02442 const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS,
02443                                          const SCEV *RHS) {
02444   SmallVector<const SCEV *, 2> Ops;
02445   Ops.push_back(LHS);
02446   Ops.push_back(RHS);
02447   return getSMaxExpr(Ops);
02448 }
02449 
02450 const SCEV *
02451 ScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
02452   assert(!Ops.empty() && "Cannot get empty smax!");
02453   if (Ops.size() == 1) return Ops[0];
02454 #ifndef NDEBUG
02455   Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
02456   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
02457     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
02458            "SCEVSMaxExpr operand types don't match!");
02459 #endif
02460 
02461   // Sort by complexity, this groups all similar expression types together.
02462   GroupByComplexity(Ops, LI);
02463 
02464   // If there are any constants, fold them together.
02465   unsigned Idx = 0;
02466   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
02467     ++Idx;
02468     assert(Idx < Ops.size());
02469     while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
02470       // We found two constants, fold them together!
02471       ConstantInt *Fold = ConstantInt::get(getContext(),
02472                               APIntOps::smax(LHSC->getValue()->getValue(),
02473                                              RHSC->getValue()->getValue()));
02474       Ops[0] = getConstant(Fold);
02475       Ops.erase(Ops.begin()+1);  // Erase the folded element
02476       if (Ops.size() == 1) return Ops[0];
02477       LHSC = cast<SCEVConstant>(Ops[0]);
02478     }
02479 
02480     // If we are left with a constant minimum-int, strip it off.
02481     if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(true)) {
02482       Ops.erase(Ops.begin());
02483       --Idx;
02484     } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(true)) {
02485       // If we have an smax with a constant maximum-int, it will always be
02486       // maximum-int.
02487       return Ops[0];
02488     }
02489 
02490     if (Ops.size() == 1) return Ops[0];
02491   }
02492 
02493   // Find the first SMax
02494   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scSMaxExpr)
02495     ++Idx;
02496 
02497   // Check to see if one of the operands is an SMax. If so, expand its operands
02498   // onto our operand list, and recurse to simplify.
02499   if (Idx < Ops.size()) {
02500     bool DeletedSMax = false;
02501     while (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(Ops[Idx])) {
02502       Ops.erase(Ops.begin()+Idx);
02503       Ops.append(SMax->op_begin(), SMax->op_end());
02504       DeletedSMax = true;
02505     }
02506 
02507     if (DeletedSMax)
02508       return getSMaxExpr(Ops);
02509   }
02510 
02511   // Okay, check to see if the same value occurs in the operand list twice.  If
02512   // so, delete one.  Since we sorted the list, these values are required to
02513   // be adjacent.
02514   for (unsigned i = 0, e = Ops.size()-1; i != e; ++i)
02515     //  X smax Y smax Y  -->  X smax Y
02516     //  X smax Y         -->  X, if X is always greater than Y
02517     if (Ops[i] == Ops[i+1] ||
02518         isKnownPredicate(ICmpInst::ICMP_SGE, Ops[i], Ops[i+1])) {
02519       Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2);
02520       --i; --e;
02521     } else if (isKnownPredicate(ICmpInst::ICMP_SLE, Ops[i], Ops[i+1])) {
02522       Ops.erase(Ops.begin()+i, Ops.begin()+i+1);
02523       --i; --e;
02524     }
02525 
02526   if (Ops.size() == 1) return Ops[0];
02527 
02528   assert(!Ops.empty() && "Reduced smax down to nothing!");
02529 
02530   // Okay, it looks like we really DO need an smax expr.  Check to see if we
02531   // already have one, otherwise create a new one.
02532   FoldingSetNodeID ID;
02533   ID.AddInteger(scSMaxExpr);
02534   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02535     ID.AddPointer(Ops[i]);
02536   void *IP = nullptr;
02537   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
02538   const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
02539   std::uninitialized_copy(Ops.begin(), Ops.end(), O);
02540   SCEV *S = new (SCEVAllocator) SCEVSMaxExpr(ID.Intern(SCEVAllocator),
02541                                              O, Ops.size());
02542   UniqueSCEVs.InsertNode(S, IP);
02543   return S;
02544 }
02545 
02546 const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS,
02547                                          const SCEV *RHS) {
02548   SmallVector<const SCEV *, 2> Ops;
02549   Ops.push_back(LHS);
02550   Ops.push_back(RHS);
02551   return getUMaxExpr(Ops);
02552 }
02553 
02554 const SCEV *
02555 ScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
02556   assert(!Ops.empty() && "Cannot get empty umax!");
02557   if (Ops.size() == 1) return Ops[0];
02558 #ifndef NDEBUG
02559   Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
02560   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
02561     assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
02562            "SCEVUMaxExpr operand types don't match!");
02563 #endif
02564 
02565   // Sort by complexity, this groups all similar expression types together.
02566   GroupByComplexity(Ops, LI);
02567 
02568   // If there are any constants, fold them together.
02569   unsigned Idx = 0;
02570   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
02571     ++Idx;
02572     assert(Idx < Ops.size());
02573     while (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Ops[Idx])) {
02574       // We found two constants, fold them together!
02575       ConstantInt *Fold = ConstantInt::get(getContext(),
02576                               APIntOps::umax(LHSC->getValue()->getValue(),
02577                                              RHSC->getValue()->getValue()));
02578       Ops[0] = getConstant(Fold);
02579       Ops.erase(Ops.begin()+1);  // Erase the folded element
02580       if (Ops.size() == 1) return Ops[0];
02581       LHSC = cast<SCEVConstant>(Ops[0]);
02582     }
02583 
02584     // If we are left with a constant minimum-int, strip it off.
02585     if (cast<SCEVConstant>(Ops[0])->getValue()->isMinValue(false)) {
02586       Ops.erase(Ops.begin());
02587       --Idx;
02588     } else if (cast<SCEVConstant>(Ops[0])->getValue()->isMaxValue(false)) {
02589       // If we have an umax with a constant maximum-int, it will always be
02590       // maximum-int.
02591       return Ops[0];
02592     }
02593 
02594     if (Ops.size() == 1) return Ops[0];
02595   }
02596 
02597   // Find the first UMax
02598   while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scUMaxExpr)
02599     ++Idx;
02600 
02601   // Check to see if one of the operands is a UMax. If so, expand its operands
02602   // onto our operand list, and recurse to simplify.
02603   if (Idx < Ops.size()) {
02604     bool DeletedUMax = false;
02605     while (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(Ops[Idx])) {
02606       Ops.erase(Ops.begin()+Idx);
02607       Ops.append(UMax->op_begin(), UMax->op_end());
02608       DeletedUMax = true;
02609     }
02610 
02611     if (DeletedUMax)
02612       return getUMaxExpr(Ops);
02613   }
02614 
02615   // Okay, check to see if the same value occurs in the operand list twice.  If
02616   // so, delete one.  Since we sorted the list, these values are required to
02617   // be adjacent.
02618   for (unsigned i = 0, e = Ops.size()-1; i != e; ++i)
02619     //  X umax Y umax Y  -->  X umax Y
02620     //  X umax Y         -->  X, if X is always greater than Y
02621     if (Ops[i] == Ops[i+1] ||
02622         isKnownPredicate(ICmpInst::ICMP_UGE, Ops[i], Ops[i+1])) {
02623       Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2);
02624       --i; --e;
02625     } else if (isKnownPredicate(ICmpInst::ICMP_ULE, Ops[i], Ops[i+1])) {
02626       Ops.erase(Ops.begin()+i, Ops.begin()+i+1);
02627       --i; --e;
02628     }
02629 
02630   if (Ops.size() == 1) return Ops[0];
02631 
02632   assert(!Ops.empty() && "Reduced umax down to nothing!");
02633 
02634   // Okay, it looks like we really DO need a umax expr.  Check to see if we
02635   // already have one, otherwise create a new one.
02636   FoldingSetNodeID ID;
02637   ID.AddInteger(scUMaxExpr);
02638   for (unsigned i = 0, e = Ops.size(); i != e; ++i)
02639     ID.AddPointer(Ops[i]);
02640   void *IP = nullptr;
02641   if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S;
02642   const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Ops.size());
02643   std::uninitialized_copy(Ops.begin(), Ops.end(), O);
02644   SCEV *S = new (SCEVAllocator) SCEVUMaxExpr(ID.Intern(SCEVAllocator),
02645                                              O, Ops.size());
02646   UniqueSCEVs.InsertNode(S, IP);
02647   return S;
02648 }
02649 
02650 const SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS,
02651                                          const SCEV *RHS) {
02652   // ~smax(~x, ~y) == smin(x, y).
02653   return getNotSCEV(getSMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS)));
02654 }
02655 
02656 const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS,
02657                                          const SCEV *RHS) {
02658   // ~umax(~x, ~y) == umin(x, y)
02659   return getNotSCEV(getUMaxExpr(getNotSCEV(LHS), getNotSCEV(RHS)));
02660 }
02661 
02662 const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) {
02663   // If we have DataLayout, we can bypass creating a target-independent
02664   // constant expression and then folding it back into a ConstantInt.
02665   // This is just a compile-time optimization.
02666   if (DL)
02667     return getConstant(IntTy, DL->getTypeAllocSize(AllocTy));
02668 
02669   Constant *C = ConstantExpr::getSizeOf(AllocTy);
02670   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
02671     if (Constant *Folded = ConstantFoldConstantExpression(CE, DL, TLI))
02672       C = Folded;
02673   Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(AllocTy));
02674   assert(Ty == IntTy && "Effective SCEV type doesn't match");
02675   return getTruncateOrZeroExtend(getSCEV(C), Ty);
02676 }
02677 
02678 const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy,
02679                                              StructType *STy,
02680                                              unsigned FieldNo) {
02681   // If we have DataLayout, we can bypass creating a target-independent
02682   // constant expression and then folding it back into a ConstantInt.
02683   // This is just a compile-time optimization.
02684   if (DL) {
02685     return getConstant(IntTy,
02686                        DL->getStructLayout(STy)->getElementOffset(FieldNo));
02687   }
02688 
02689   Constant *C = ConstantExpr::getOffsetOf(STy, FieldNo);
02690   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
02691     if (Constant *Folded = ConstantFoldConstantExpression(CE, DL, TLI))
02692       C = Folded;
02693 
02694   Type *Ty = getEffectiveSCEVType(PointerType::getUnqual(STy));
02695   return getTruncateOrZeroExtend(getSCEV(C), Ty);
02696 }
02697 
02698 const SCEV *ScalarEvolution::getUnknown(Value *V) {
02699   // Don't attempt to do anything other than create a SCEVUnknown object
02700   // here.  createSCEV only calls getUnknown after checking for all other
02701   // interesting possibilities, and any other code that calls getUnknown
02702   // is doing so in order to hide a value from SCEV canonicalization.
02703 
02704   FoldingSetNodeID ID;
02705   ID.AddInteger(scUnknown);
02706   ID.AddPointer(V);
02707   void *IP = nullptr;
02708   if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) {
02709     assert(cast<SCEVUnknown>(S)->getValue() == V &&
02710            "Stale SCEVUnknown in uniquing map!");
02711     return S;
02712   }
02713   SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this,
02714                                             FirstUnknown);
02715   FirstUnknown = cast<SCEVUnknown>(S);
02716   UniqueSCEVs.InsertNode(S, IP);
02717   return S;
02718 }
02719 
02720 //===----------------------------------------------------------------------===//
02721 //            Basic SCEV Analysis and PHI Idiom Recognition Code
02722 //
02723 
02724 /// isSCEVable - Test if values of the given type are analyzable within
02725 /// the SCEV framework. This primarily includes integer types, and it
02726 /// can optionally include pointer types if the ScalarEvolution class
02727 /// has access to target-specific information.
02728 bool ScalarEvolution::isSCEVable(Type *Ty) const {
02729   // Integers and pointers are always SCEVable.
02730   return Ty->isIntegerTy() || Ty->isPointerTy();
02731 }
02732 
02733 /// getTypeSizeInBits - Return the size in bits of the specified type,
02734 /// for which isSCEVable must return true.
02735 uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
02736   assert(isSCEVable(Ty) && "Type is not SCEVable!");
02737 
02738   // If we have a DataLayout, use it!
02739   if (DL)
02740     return DL->getTypeSizeInBits(Ty);
02741 
02742   // Integer types have fixed sizes.
02743   if (Ty->isIntegerTy())
02744     return Ty->getPrimitiveSizeInBits();
02745 
02746   // The only other support type is pointer. Without DataLayout, conservatively
02747   // assume pointers are 64-bit.
02748   assert(Ty->isPointerTy() && "isSCEVable permitted a non-SCEVable type!");
02749   return 64;
02750 }
02751 
02752 /// getEffectiveSCEVType - Return a type with the same bitwidth as
02753 /// the given type and which represents how SCEV will treat the given
02754 /// type, for which isSCEVable must return true. For pointer types,
02755 /// this is the pointer-sized integer type.
02756 Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
02757   assert(isSCEVable(Ty) && "Type is not SCEVable!");
02758 
02759   if (Ty->isIntegerTy()) {
02760     return Ty;
02761   }
02762 
02763   // The only other support type is pointer.
02764   assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
02765 
02766   if (DL)
02767     return DL->getIntPtrType(Ty);
02768 
02769   // Without DataLayout, conservatively assume pointers are 64-bit.
02770   return Type::getInt64Ty(getContext());
02771 }
02772 
02773 const SCEV *ScalarEvolution::getCouldNotCompute() {
02774   return &CouldNotCompute;
02775 }
02776 
02777 namespace {
02778   // Helper class working with SCEVTraversal to figure out if a SCEV contains
02779   // a SCEVUnknown with null value-pointer. FindInvalidSCEVUnknown::FindOne
02780   // is set iff if find such SCEVUnknown.
02781   //
02782   struct FindInvalidSCEVUnknown {
02783     bool FindOne;
02784     FindInvalidSCEVUnknown() { FindOne = false; }
02785     bool follow(const SCEV *S) {
02786       switch (static_cast<SCEVTypes>(S->getSCEVType())) {
02787       case scConstant:
02788         return false;
02789       case scUnknown:
02790         if (!cast<SCEVUnknown>(S)->getValue())
02791           FindOne = true;
02792         return false;
02793       default:
02794         return true;
02795       }
02796     }
02797     bool isDone() const { return FindOne; }
02798   };
02799 }
02800 
02801 bool ScalarEvolution::checkValidity(const SCEV *S) const {
02802   FindInvalidSCEVUnknown F;
02803   SCEVTraversal<FindInvalidSCEVUnknown> ST(F);
02804   ST.visitAll(S);
02805 
02806   return !F.FindOne;
02807 }
02808 
02809 /// getSCEV - Return an existing SCEV if it exists, otherwise analyze the
02810 /// expression and create a new one.
02811 const SCEV *ScalarEvolution::getSCEV(Value *V) {
02812   assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
02813 
02814   ValueExprMapType::iterator I = ValueExprMap.find_as(V);
02815   if (I != ValueExprMap.end()) {
02816     const SCEV *S = I->second;
02817     if (checkValidity(S))
02818       return S;
02819     else
02820       ValueExprMap.erase(I);
02821   }
02822   const SCEV *S = createSCEV(V);
02823 
02824   // The process of creating a SCEV for V may have caused other SCEVs
02825   // to have been created, so it's necessary to insert the new entry
02826   // from scratch, rather than trying to remember the insert position
02827   // above.
02828   ValueExprMap.insert(std::make_pair(SCEVCallbackVH(V, this), S));
02829   return S;
02830 }
02831 
02832 /// getNegativeSCEV - Return a SCEV corresponding to -V = -1*V
02833 ///
02834 const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V) {
02835   if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
02836     return getConstant(
02837                cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue())));
02838 
02839   Type *Ty = V->getType();
02840   Ty = getEffectiveSCEVType(Ty);
02841   return getMulExpr(V,
02842                   getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty))));
02843 }
02844 
02845 /// getNotSCEV - Return a SCEV corresponding to ~V = -1-V
02846 const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
02847   if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
02848     return getConstant(
02849                 cast<ConstantInt>(ConstantExpr::getNot(VC->getValue())));
02850 
02851   Type *Ty = V->getType();
02852   Ty = getEffectiveSCEVType(Ty);
02853   const SCEV *AllOnes =
02854                    getConstant(cast<ConstantInt>(Constant::getAllOnesValue(Ty)));
02855   return getMinusSCEV(AllOnes, V);
02856 }
02857 
02858 /// getMinusSCEV - Return LHS-RHS.  Minus is represented in SCEV as A+B*-1.
02859 const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS,
02860                                           SCEV::NoWrapFlags Flags) {
02861   assert(!maskFlags(Flags, SCEV::FlagNUW) && "subtraction does not have NUW");
02862 
02863   // Fast path: X - X --> 0.
02864   if (LHS == RHS)
02865     return getConstant(LHS->getType(), 0);
02866 
02867   // X - Y --> X + -Y
02868   return getAddExpr(LHS, getNegativeSCEV(RHS), Flags);
02869 }
02870 
02871 /// getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the
02872 /// input value to the specified type.  If the type must be extended, it is zero
02873 /// extended.
02874 const SCEV *
02875 ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty) {
02876   Type *SrcTy = V->getType();
02877   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
02878          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
02879          "Cannot truncate or zero extend with non-integer arguments!");
02880   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
02881     return V;  // No conversion
02882   if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
02883     return getTruncateExpr(V, Ty);
02884   return getZeroExtendExpr(V, Ty);
02885 }
02886 
02887 /// getTruncateOrSignExtend - Return a SCEV corresponding to a conversion of the
02888 /// input value to the specified type.  If the type must be extended, it is sign
02889 /// extended.
02890 const SCEV *
02891 ScalarEvolution::getTruncateOrSignExtend(const SCEV *V,
02892                                          Type *Ty) {
02893   Type *SrcTy = V->getType();
02894   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
02895          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
02896          "Cannot truncate or zero extend with non-integer arguments!");
02897   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
02898     return V;  // No conversion
02899   if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
02900     return getTruncateExpr(V, Ty);
02901   return getSignExtendExpr(V, Ty);
02902 }
02903 
02904 /// getNoopOrZeroExtend - Return a SCEV corresponding to a conversion of the
02905 /// input value to the specified type.  If the type must be extended, it is zero
02906 /// extended.  The conversion must not be narrowing.
02907 const SCEV *
02908 ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
02909   Type *SrcTy = V->getType();
02910   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
02911          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
02912          "Cannot noop or zero extend with non-integer arguments!");
02913   assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
02914          "getNoopOrZeroExtend cannot truncate!");
02915   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
02916     return V;  // No conversion
02917   return getZeroExtendExpr(V, Ty);
02918 }
02919 
02920 /// getNoopOrSignExtend - Return a SCEV corresponding to a conversion of the
02921 /// input value to the specified type.  If the type must be extended, it is sign
02922 /// extended.  The conversion must not be narrowing.
02923 const SCEV *
02924 ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
02925   Type *SrcTy = V->getType();
02926   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
02927          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
02928          "Cannot noop or sign extend with non-integer arguments!");
02929   assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
02930          "getNoopOrSignExtend cannot truncate!");
02931   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
02932     return V;  // No conversion
02933   return getSignExtendExpr(V, Ty);
02934 }
02935 
02936 /// getNoopOrAnyExtend - Return a SCEV corresponding to a conversion of
02937 /// the input value to the specified type. If the type must be extended,
02938 /// it is extended with unspecified bits. The conversion must not be
02939 /// narrowing.
02940 const SCEV *
02941 ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
02942   Type *SrcTy = V->getType();
02943   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
02944          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
02945          "Cannot noop or any extend with non-integer arguments!");
02946   assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
02947          "getNoopOrAnyExtend cannot truncate!");
02948   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
02949     return V;  // No conversion
02950   return getAnyExtendExpr(V, Ty);
02951 }
02952 
02953 /// getTruncateOrNoop - Return a SCEV corresponding to a conversion of the
02954 /// input value to the specified type.  The conversion must not be widening.
02955 const SCEV *
02956 ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
02957   Type *SrcTy = V->getType();
02958   assert((SrcTy->isIntegerTy() || SrcTy->isPointerTy()) &&
02959          (Ty->isIntegerTy() || Ty->isPointerTy()) &&
02960          "Cannot truncate or noop with non-integer arguments!");
02961   assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&
02962          "getTruncateOrNoop cannot extend!");
02963   if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
02964     return V;  // No conversion
02965   return getTruncateExpr(V, Ty);
02966 }
02967 
02968 /// getUMaxFromMismatchedTypes - Promote the operands to the wider of
02969 /// the types using zero-extension, and then perform a umax operation
02970 /// with them.
02971 const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
02972                                                         const SCEV *RHS) {
02973   const SCEV *PromotedLHS = LHS;
02974   const SCEV *PromotedRHS = RHS;
02975 
02976   if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType()))
02977     PromotedRHS = getZeroExtendExpr(RHS, LHS->getType());
02978   else
02979     PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType());
02980 
02981   return getUMaxExpr(PromotedLHS, PromotedRHS);
02982 }
02983 
02984 /// getUMinFromMismatchedTypes - Promote the operands to the wider of
02985 /// the types using zero-extension, and then perform a umin operation
02986 /// with them.
02987 const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
02988                                                         const SCEV *RHS) {
02989   const SCEV *PromotedLHS = LHS;
02990   const SCEV *PromotedRHS = RHS;
02991 
02992   if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType()))
02993     PromotedRHS = getZeroExtendExpr(RHS, LHS->getType());
02994   else
02995     PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType());
02996 
02997   return getUMinExpr(PromotedLHS, PromotedRHS);
02998 }
02999 
03000 /// getPointerBase - Transitively follow the chain of pointer-type operands
03001 /// until reaching a SCEV that does not have a single pointer operand. This
03002 /// returns a SCEVUnknown pointer for well-formed pointer-type expressions,
03003 /// but corner cases do exist.
03004 const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
03005   // A pointer operand may evaluate to a nonpointer expression, such as null.
03006   if (!V->getType()->isPointerTy())
03007     return V;
03008 
03009   if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(V)) {
03010     return getPointerBase(Cast->getOperand());
03011   }
03012   else if (const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(V)) {
03013     const SCEV *PtrOp = nullptr;
03014     for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end();
03015          I != E; ++I) {
03016       if ((*I)->getType()->isPointerTy()) {
03017         // Cannot find the base of an expression with multiple pointer operands.
03018         if (PtrOp)
03019           return V;
03020         PtrOp = *I;
03021       }
03022     }
03023     if (!PtrOp)
03024       return V;
03025     return getPointerBase(PtrOp);
03026   }
03027   return V;
03028 }
03029 
03030 /// PushDefUseChildren - Push users of the given Instruction
03031 /// onto the given Worklist.
03032 static void
03033 PushDefUseChildren(Instruction *I,
03034                    SmallVectorImpl<Instruction *> &Worklist) {
03035   // Push the def-use children onto the Worklist stack.
03036   for (User *U : I->users())
03037     Worklist.push_back(cast<Instruction>(U));
03038 }
03039 
03040 /// ForgetSymbolicValue - This looks up computed SCEV values for all
03041 /// instructions that depend on the given instruction and removes them from
03042 /// the ValueExprMapType map if they reference SymName. This is used during PHI
03043 /// resolution.
03044 void
03045 ScalarEvolution::ForgetSymbolicName(Instruction *PN, const SCEV *SymName) {
03046   SmallVector<Instruction *, 16> Worklist;
03047   PushDefUseChildren(PN, Worklist);
03048 
03049   SmallPtrSet<Instruction *, 8> Visited;
03050   Visited.insert(PN);
03051   while (!Worklist.empty()) {
03052     Instruction *I = Worklist.pop_back_val();
03053     if (!Visited.insert(I)) continue;
03054 
03055     ValueExprMapType::iterator It =
03056       ValueExprMap.find_as(static_cast<Value *>(I));
03057     if (It != ValueExprMap.end()) {
03058       const SCEV *Old = It->second;
03059 
03060       // Short-circuit the def-use traversal if the symbolic name
03061       // ceases to appear in expressions.
03062       if (Old != SymName && !hasOperand(Old, SymName))
03063         continue;
03064 
03065       // SCEVUnknown for a PHI either means that it has an unrecognized
03066       // structure, it's a PHI that's in the progress of being computed
03067       // by createNodeForPHI, or it's a single-value PHI. In the first case,
03068       // additional loop trip count information isn't going to change anything.
03069       // In the second case, createNodeForPHI will perform the necessary
03070       // updates on its own when it gets to that point. In the third, we do
03071       // want to forget the SCEVUnknown.
03072       if (!isa<PHINode>(I) ||
03073           !isa<SCEVUnknown>(Old) ||
03074           (I != PN && Old == SymName)) {
03075         forgetMemoizedResults(Old);
03076         ValueExprMap.erase(It);
03077       }
03078     }
03079 
03080     PushDefUseChildren(I, Worklist);
03081   }
03082 }
03083 
03084 /// createNodeForPHI - PHI nodes have two cases.  Either the PHI node exists in
03085 /// a loop header, making it a potential recurrence, or it doesn't.
03086 ///
03087 const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
03088   if (const Loop *L = LI->getLoopFor(PN->getParent()))
03089     if (L->getHeader() == PN->getParent()) {
03090       // The loop may have multiple entrances or multiple exits; we can analyze
03091       // this phi as an addrec if it has a unique entry value and a unique
03092       // backedge value.
03093       Value *BEValueV = nullptr, *StartValueV = nullptr;
03094       for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
03095         Value *V = PN->getIncomingValue(i);
03096         if (L->contains(PN->getIncomingBlock(i))) {
03097           if (!BEValueV) {
03098             BEValueV = V;
03099           } else if (BEValueV != V) {
03100             BEValueV = nullptr;
03101             break;
03102           }
03103         } else if (!StartValueV) {
03104           StartValueV = V;
03105         } else if (StartValueV != V) {
03106           StartValueV = nullptr;
03107           break;
03108         }
03109       }
03110       if (BEValueV && StartValueV) {
03111         // While we are analyzing this PHI node, handle its value symbolically.
03112         const SCEV *SymbolicName = getUnknown(PN);
03113         assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
03114                "PHI node already processed?");
03115         ValueExprMap.insert(std::make_pair(SCEVCallbackVH(PN, this), SymbolicName));
03116 
03117         // Using this symbolic name for the PHI, analyze the value coming around
03118         // the back-edge.
03119         const SCEV *BEValue = getSCEV(BEValueV);
03120 
03121         // NOTE: If BEValue is loop invariant, we know that the PHI node just
03122         // has a special value for the first iteration of the loop.
03123 
03124         // If the value coming around the backedge is an add with the symbolic
03125         // value we just inserted, then we found a simple induction variable!
03126         if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) {
03127           // If there is a single occurrence of the symbolic value, replace it
03128           // with a recurrence.
03129           unsigned FoundIndex = Add->getNumOperands();
03130           for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
03131             if (Add->getOperand(i) == SymbolicName)
03132               if (FoundIndex == e) {
03133                 FoundIndex = i;
03134                 break;
03135               }
03136 
03137           if (FoundIndex != Add->getNumOperands()) {
03138             // Create an add with everything but the specified operand.
03139             SmallVector<const SCEV *, 8> Ops;
03140             for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
03141               if (i != FoundIndex)
03142                 Ops.push_back(Add->getOperand(i));
03143             const SCEV *Accum = getAddExpr(Ops);
03144 
03145             // This is not a valid addrec if the step amount is varying each
03146             // loop iteration, but is not itself an addrec in this loop.
03147             if (isLoopInvariant(Accum, L) ||
03148                 (isa<SCEVAddRecExpr>(Accum) &&
03149                  cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) {
03150               SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
03151 
03152               // If the increment doesn't overflow, then neither the addrec nor
03153               // the post-increment will overflow.
03154               if (const AddOperator *OBO = dyn_cast<AddOperator>(BEValueV)) {
03155                 if (OBO->hasNoUnsignedWrap())
03156                   Flags = setFlags(Flags, SCEV::FlagNUW);
03157                 if (OBO->hasNoSignedWrap())
03158                   Flags = setFlags(Flags, SCEV::FlagNSW);
03159               } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(BEValueV)) {
03160                 // If the increment is an inbounds GEP, then we know the address
03161                 // space cannot be wrapped around. We cannot make any guarantee
03162                 // about signed or unsigned overflow because pointers are
03163                 // unsigned but we may have a negative index from the base
03164                 // pointer. We can guarantee that no unsigned wrap occurs if the
03165                 // indices form a positive value.
03166                 if (GEP->isInBounds()) {
03167                   Flags = setFlags(Flags, SCEV::FlagNW);
03168 
03169                   const SCEV *Ptr = getSCEV(GEP->getPointerOperand());
03170                   if (isKnownPositive(getMinusSCEV(getSCEV(GEP), Ptr)))
03171                     Flags = setFlags(Flags, SCEV::FlagNUW);
03172                 }
03173               } else if (const SubOperator *OBO =
03174                            dyn_cast<SubOperator>(BEValueV)) {
03175                 if (OBO->hasNoUnsignedWrap())
03176                   Flags = setFlags(Flags, SCEV::FlagNUW);
03177                 if (OBO->hasNoSignedWrap())
03178                   Flags = setFlags(Flags, SCEV::FlagNSW);
03179               }
03180 
03181               const SCEV *StartVal = getSCEV(StartValueV);
03182               const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags);
03183 
03184               // Since the no-wrap flags are on the increment, they apply to the
03185               // post-incremented value as well.
03186               if (isLoopInvariant(Accum, L))
03187                 (void)getAddRecExpr(getAddExpr(StartVal, Accum),
03188                                     Accum, L, Flags);
03189 
03190               // Okay, for the entire analysis of this edge we assumed the PHI
03191               // to be symbolic.  We now need to go back and purge all of the
03192               // entries for the scalars that use the symbolic expression.
03193               ForgetSymbolicName(PN, SymbolicName);
03194               ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV;
03195               return PHISCEV;
03196             }
03197           }
03198         } else if (const SCEVAddRecExpr *AddRec =
03199                      dyn_cast<SCEVAddRecExpr>(BEValue)) {
03200           // Otherwise, this could be a loop like this:
03201           //     i = 0;  for (j = 1; ..; ++j) { ....  i = j; }
03202           // In this case, j = {1,+,1}  and BEValue is j.
03203           // Because the other in-value of i (0) fits the evolution of BEValue
03204           // i really is an addrec evolution.
03205           if (AddRec->getLoop() == L && AddRec->isAffine()) {
03206             const SCEV *StartVal = getSCEV(StartValueV);
03207 
03208             // If StartVal = j.start - j.stride, we can use StartVal as the
03209             // initial step of the addrec evolution.
03210             if (StartVal == getMinusSCEV(AddRec->getOperand(0),
03211                                          AddRec->getOperand(1))) {
03212               // FIXME: For constant StartVal, we should be able to infer
03213               // no-wrap flags.
03214               const SCEV *PHISCEV =
03215                 getAddRecExpr(StartVal, AddRec->getOperand(1), L,
03216                               SCEV::FlagAnyWrap);
03217 
03218               // Okay, for the entire analysis of this edge we assumed the PHI
03219               // to be symbolic.  We now need to go back and purge all of the
03220               // entries for the scalars that use the symbolic expression.
03221               ForgetSymbolicName(PN, SymbolicName);
03222               ValueExprMap[SCEVCallbackVH(PN, this)] = PHISCEV;
03223               return PHISCEV;
03224             }
03225           }
03226         }
03227       }
03228     }
03229 
03230   // If the PHI has a single incoming value, follow that value, unless the
03231   // PHI's incoming blocks are in a different loop, in which case doing so
03232   // risks breaking LCSSA form. Instcombine would normally zap these, but
03233   // it doesn't have DominatorTree information, so it may miss cases.
03234   if (Value *V = SimplifyInstruction(PN, DL, TLI, DT))
03235     if (LI->replacementPreservesLCSSAForm(PN, V))
03236       return getSCEV(V);
03237 
03238   // If it's not a loop phi, we can't handle it yet.
03239   return getUnknown(PN);
03240 }
03241 
03242 /// createNodeForGEP - Expand GEP instructions into add and multiply
03243 /// operations. This allows them to be analyzed by regular SCEV code.
03244 ///
03245 const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
03246   Type *IntPtrTy = getEffectiveSCEVType(GEP->getType());
03247   Value *Base = GEP->getOperand(0);
03248   // Don't attempt to analyze GEPs over unsized objects.
03249   if (!Base->getType()->getPointerElementType()->isSized())
03250     return getUnknown(GEP);
03251 
03252   // Don't blindly transfer the inbounds flag from the GEP instruction to the
03253   // Add expression, because the Instruction may be guarded by control flow
03254   // and the no-overflow bits may not be valid for the expression in any
03255   // context.
03256   SCEV::NoWrapFlags Wrap = GEP->isInBounds() ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
03257 
03258   const SCEV *TotalOffset = getConstant(IntPtrTy, 0);
03259   gep_type_iterator GTI = gep_type_begin(GEP);
03260   for (GetElementPtrInst::op_iterator I = std::next(GEP->op_begin()),
03261                                       E = GEP->op_end();
03262        I != E; ++I) {
03263     Value *Index = *I;
03264     // Compute the (potentially symbolic) offset in bytes for this index.
03265     if (StructType *STy = dyn_cast<StructType>(*GTI++)) {
03266       // For a struct, add the member offset.
03267       unsigned FieldNo = cast<ConstantInt>(Index)->getZExtValue();
03268       const SCEV *FieldOffset = getOffsetOfExpr(IntPtrTy, STy, FieldNo);
03269 
03270       // Add the field offset to the running total offset.
03271       TotalOffset = getAddExpr(TotalOffset, FieldOffset);
03272     } else {
03273       // For an array, add the element offset, explicitly scaled.
03274       const SCEV *ElementSize = getSizeOfExpr(IntPtrTy, *GTI);
03275       const SCEV *IndexS = getSCEV(Index);
03276       // Getelementptr indices are signed.
03277       IndexS = getTruncateOrSignExtend(IndexS, IntPtrTy);
03278 
03279       // Multiply the index by the element size to compute the element offset.
03280       const SCEV *LocalOffset = getMulExpr(IndexS, ElementSize, Wrap);
03281 
03282       // Add the element offset to the running total offset.
03283       TotalOffset = getAddExpr(TotalOffset, LocalOffset);
03284     }
03285   }
03286 
03287   // Get the SCEV for the GEP base.
03288   const SCEV *BaseS = getSCEV(Base);
03289 
03290   // Add the total offset from all the GEP indices to the base.
03291   return getAddExpr(BaseS, TotalOffset, Wrap);
03292 }
03293 
03294 /// GetMinTrailingZeros - Determine the minimum number of zero bits that S is
03295 /// guaranteed to end in (at every loop iteration).  It is, at the same time,
03296 /// the minimum number of times S is divisible by 2.  For example, given {4,+,8}
03297 /// it returns 2.  If S is guaranteed to be 0, it returns the bitwidth of S.
03298 uint32_t
03299 ScalarEvolution::GetMinTrailingZeros(const SCEV *S) {
03300   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
03301     return C->getValue()->getValue().countTrailingZeros();
03302 
03303   if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(S))
03304     return std::min(GetMinTrailingZeros(T->getOperand()),
03305                     (uint32_t)getTypeSizeInBits(T->getType()));
03306 
03307   if (const SCEVZeroExtendExpr *E = dyn_cast<SCEVZeroExtendExpr>(S)) {
03308     uint32_t OpRes = GetMinTrailingZeros(E->getOperand());
03309     return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ?
03310              getTypeSizeInBits(E->getType()) : OpRes;
03311   }
03312 
03313   if (const SCEVSignExtendExpr *E = dyn_cast<SCEVSignExtendExpr>(S)) {
03314     uint32_t OpRes = GetMinTrailingZeros(E->getOperand());
03315     return OpRes == getTypeSizeInBits(E->getOperand()->getType()) ?
03316              getTypeSizeInBits(E->getType()) : OpRes;
03317   }
03318 
03319   if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(S)) {
03320     // The result is the min of all operands results.
03321     uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0));
03322     for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i)
03323       MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i)));
03324     return MinOpRes;
03325   }
03326 
03327   if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) {
03328     // The result is the sum of all operands results.
03329     uint32_t SumOpRes = GetMinTrailingZeros(M->getOperand(0));
03330     uint32_t BitWidth = getTypeSizeInBits(M->getType());
03331     for (unsigned i = 1, e = M->getNumOperands();
03332          SumOpRes != BitWidth && i != e; ++i)
03333       SumOpRes = std::min(SumOpRes + GetMinTrailingZeros(M->getOperand(i)),
03334                           BitWidth);
03335     return SumOpRes;
03336   }
03337 
03338   if (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(S)) {
03339     // The result is the min of all operands results.
03340     uint32_t MinOpRes = GetMinTrailingZeros(A->getOperand(0));
03341     for (unsigned i = 1, e = A->getNumOperands(); MinOpRes && i != e; ++i)
03342       MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(A->getOperand(i)));
03343     return MinOpRes;
03344   }
03345 
03346   if (const SCEVSMaxExpr *M = dyn_cast<SCEVSMaxExpr>(S)) {
03347     // The result is the min of all operands results.
03348     uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0));
03349     for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i)
03350       MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i)));
03351     return MinOpRes;
03352   }
03353 
03354   if (const SCEVUMaxExpr *M = dyn_cast<SCEVUMaxExpr>(S)) {
03355     // The result is the min of all operands results.
03356     uint32_t MinOpRes = GetMinTrailingZeros(M->getOperand(0));
03357     for (unsigned i = 1, e = M->getNumOperands(); MinOpRes && i != e; ++i)
03358       MinOpRes = std::min(MinOpRes, GetMinTrailingZeros(M->getOperand(i)));
03359     return MinOpRes;
03360   }
03361 
03362   if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
03363     // For a SCEVUnknown, ask ValueTracking.
03364     unsigned BitWidth = getTypeSizeInBits(U->getType());
03365     APInt Zeros(BitWidth, 0), Ones(BitWidth, 0);
03366     ComputeMaskedBits(U->getValue(), Zeros, Ones);
03367     return Zeros.countTrailingOnes();
03368   }
03369 
03370   // SCEVUDivExpr
03371   return 0;
03372 }
03373 
03374 /// getUnsignedRange - Determine the unsigned range for a particular SCEV.
03375 ///
03376 ConstantRange
03377 ScalarEvolution::getUnsignedRange(const SCEV *S) {
03378   // See if we've computed this range already.
03379   DenseMap<const SCEV *, ConstantRange>::iterator I = UnsignedRanges.find(S);
03380   if (I != UnsignedRanges.end())
03381     return I->second;
03382 
03383   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
03384     return setUnsignedRange(C, ConstantRange(C->getValue()->getValue()));
03385 
03386   unsigned BitWidth = getTypeSizeInBits(S->getType());
03387   ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
03388 
03389   // If the value has known zeros, the maximum unsigned value will have those
03390   // known zeros as well.
03391   uint32_t TZ = GetMinTrailingZeros(S);
03392   if (TZ != 0)
03393     ConservativeResult =
03394       ConstantRange(APInt::getMinValue(BitWidth),
03395                     APInt::getMaxValue(BitWidth).lshr(TZ).shl(TZ) + 1);
03396 
03397   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
03398     ConstantRange X = getUnsignedRange(Add->getOperand(0));
03399     for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i)
03400       X = X.add(getUnsignedRange(Add->getOperand(i)));
03401     return setUnsignedRange(Add, ConservativeResult.intersectWith(X));
03402   }
03403 
03404   if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
03405     ConstantRange X = getUnsignedRange(Mul->getOperand(0));
03406     for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i)
03407       X = X.multiply(getUnsignedRange(Mul->getOperand(i)));
03408     return setUnsignedRange(Mul, ConservativeResult.intersectWith(X));
03409   }
03410 
03411   if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) {
03412     ConstantRange X = getUnsignedRange(SMax->getOperand(0));
03413     for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i)
03414       X = X.smax(getUnsignedRange(SMax->getOperand(i)));
03415     return setUnsignedRange(SMax, ConservativeResult.intersectWith(X));
03416   }
03417 
03418   if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) {
03419     ConstantRange X = getUnsignedRange(UMax->getOperand(0));
03420     for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i)
03421       X = X.umax(getUnsignedRange(UMax->getOperand(i)));
03422     return setUnsignedRange(UMax, ConservativeResult.intersectWith(X));
03423   }
03424 
03425   if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) {
03426     ConstantRange X = getUnsignedRange(UDiv->getLHS());
03427     ConstantRange Y = getUnsignedRange(UDiv->getRHS());
03428     return setUnsignedRange(UDiv, ConservativeResult.intersectWith(X.udiv(Y)));
03429   }
03430 
03431   if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) {
03432     ConstantRange X = getUnsignedRange(ZExt->getOperand());
03433     return setUnsignedRange(ZExt,
03434       ConservativeResult.intersectWith(X.zeroExtend(BitWidth)));
03435   }
03436 
03437   if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) {
03438     ConstantRange X = getUnsignedRange(SExt->getOperand());
03439     return setUnsignedRange(SExt,
03440       ConservativeResult.intersectWith(X.signExtend(BitWidth)));
03441   }
03442 
03443   if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) {
03444     ConstantRange X = getUnsignedRange(Trunc->getOperand());
03445     return setUnsignedRange(Trunc,
03446       ConservativeResult.intersectWith(X.truncate(BitWidth)));
03447   }
03448 
03449   if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
03450     // If there's no unsigned wrap, the value will never be less than its
03451     // initial value.
03452     if (AddRec->getNoWrapFlags(SCEV::FlagNUW))
03453       if (const SCEVConstant *C = dyn_cast<SCEVConstant>(AddRec->getStart()))
03454         if (!C->getValue()->isZero())
03455           ConservativeResult =
03456             ConservativeResult.intersectWith(
03457               ConstantRange(C->getValue()->getValue(), APInt(BitWidth, 0)));
03458 
03459     // TODO: non-affine addrec
03460     if (AddRec->isAffine()) {
03461       Type *Ty = AddRec->getType();
03462       const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop());
03463       if (!isa<SCEVCouldNotCompute>(MaxBECount) &&
03464           getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) {
03465         MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty);
03466 
03467         const SCEV *Start = AddRec->getStart();
03468         const SCEV *Step = AddRec->getStepRecurrence(*this);
03469 
03470         ConstantRange StartRange = getUnsignedRange(Start);
03471         ConstantRange StepRange = getSignedRange(Step);
03472         ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount);
03473         ConstantRange EndRange =
03474           StartRange.add(MaxBECountRange.multiply(StepRange));
03475 
03476         // Check for overflow. This must be done with ConstantRange arithmetic
03477         // because we could be called from within the ScalarEvolution overflow
03478         // checking code.
03479         ConstantRange ExtStartRange = StartRange.zextOrTrunc(BitWidth*2+1);
03480         ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1);
03481         ConstantRange ExtMaxBECountRange =
03482           MaxBECountRange.zextOrTrunc(BitWidth*2+1);
03483         ConstantRange ExtEndRange = EndRange.zextOrTrunc(BitWidth*2+1);
03484         if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) !=
03485             ExtEndRange)
03486           return setUnsignedRange(AddRec, ConservativeResult);
03487 
03488         APInt Min = APIntOps::umin(StartRange.getUnsignedMin(),
03489                                    EndRange.getUnsignedMin());
03490         APInt Max = APIntOps::umax(StartRange.getUnsignedMax(),
03491                                    EndRange.getUnsignedMax());
03492         if (Min.isMinValue() && Max.isMaxValue())
03493           return setUnsignedRange(AddRec, ConservativeResult);
03494         return setUnsignedRange(AddRec,
03495           ConservativeResult.intersectWith(ConstantRange(Min, Max+1)));
03496       }
03497     }
03498 
03499     return setUnsignedRange(AddRec, ConservativeResult);
03500   }
03501 
03502   if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
03503     // For a SCEVUnknown, ask ValueTracking.
03504     APInt Zeros(BitWidth, 0), Ones(BitWidth, 0);
03505     ComputeMaskedBits(U->getValue(), Zeros, Ones, DL);
03506     if (Ones == ~Zeros + 1)
03507       return setUnsignedRange(U, ConservativeResult);
03508     return setUnsignedRange(U,
03509       ConservativeResult.intersectWith(ConstantRange(Ones, ~Zeros + 1)));
03510   }
03511 
03512   return setUnsignedRange(S, ConservativeResult);
03513 }
03514 
03515 /// getSignedRange - Determine the signed range for a particular SCEV.
03516 ///
03517 ConstantRange
03518 ScalarEvolution::getSignedRange(const SCEV *S) {
03519   // See if we've computed this range already.
03520   DenseMap<const SCEV *, ConstantRange>::iterator I = SignedRanges.find(S);
03521   if (I != SignedRanges.end())
03522     return I->second;
03523 
03524   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
03525     return setSignedRange(C, ConstantRange(C->getValue()->getValue()));
03526 
03527   unsigned BitWidth = getTypeSizeInBits(S->getType());
03528   ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
03529 
03530   // If the value has known zeros, the maximum signed value will have those
03531   // known zeros as well.
03532   uint32_t TZ = GetMinTrailingZeros(S);
03533   if (TZ != 0)
03534     ConservativeResult =
03535       ConstantRange(APInt::getSignedMinValue(BitWidth),
03536                     APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1);
03537 
03538   if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
03539     ConstantRange X = getSignedRange(Add->getOperand(0));
03540     for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i)
03541       X = X.add(getSignedRange(Add->getOperand(i)));
03542     return setSignedRange(Add, ConservativeResult.intersectWith(X));
03543   }
03544 
03545   if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
03546     ConstantRange X = getSignedRange(Mul->getOperand(0));
03547     for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i)
03548       X = X.multiply(getSignedRange(Mul->getOperand(i)));
03549     return setSignedRange(Mul, ConservativeResult.intersectWith(X));
03550   }
03551 
03552   if (const SCEVSMaxExpr *SMax = dyn_cast<SCEVSMaxExpr>(S)) {
03553     ConstantRange X = getSignedRange(SMax->getOperand(0));
03554     for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i)
03555       X = X.smax(getSignedRange(SMax->getOperand(i)));
03556     return setSignedRange(SMax, ConservativeResult.intersectWith(X));
03557   }
03558 
03559   if (const SCEVUMaxExpr *UMax = dyn_cast<SCEVUMaxExpr>(S)) {
03560     ConstantRange X = getSignedRange(UMax->getOperand(0));
03561     for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i)
03562       X = X.umax(getSignedRange(UMax->getOperand(i)));
03563     return setSignedRange(UMax, ConservativeResult.intersectWith(X));
03564   }
03565 
03566   if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) {
03567     ConstantRange X = getSignedRange(UDiv->getLHS());
03568     ConstantRange Y = getSignedRange(UDiv->getRHS());
03569     return setSignedRange(UDiv, ConservativeResult.intersectWith(X.udiv(Y)));
03570   }
03571 
03572   if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(S)) {
03573     ConstantRange X = getSignedRange(ZExt->getOperand());
03574     return setSignedRange(ZExt,
03575       ConservativeResult.intersectWith(X.zeroExtend(BitWidth)));
03576   }
03577 
03578   if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(S)) {
03579     ConstantRange X = getSignedRange(SExt->getOperand());
03580     return setSignedRange(SExt,
03581       ConservativeResult.intersectWith(X.signExtend(BitWidth)));
03582   }
03583 
03584   if (const SCEVTruncateExpr *Trunc = dyn_cast<SCEVTruncateExpr>(S)) {
03585     ConstantRange X = getSignedRange(Trunc->getOperand());
03586     return setSignedRange(Trunc,
03587       ConservativeResult.intersectWith(X.truncate(BitWidth)));
03588   }
03589 
03590   if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
03591     // If there's no signed wrap, and all the operands have the same sign or
03592     // zero, the value won't ever change sign.
03593     if (AddRec->getNoWrapFlags(SCEV::FlagNSW)) {
03594       bool AllNonNeg = true;
03595       bool AllNonPos = true;
03596       for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
03597         if (!isKnownNonNegative(AddRec->getOperand(i))) AllNonNeg = false;
03598         if (!isKnownNonPositive(AddRec->getOperand(i))) AllNonPos = false;
03599       }
03600       if (AllNonNeg)
03601         ConservativeResult = ConservativeResult.intersectWith(
03602           ConstantRange(APInt(BitWidth, 0),
03603                         APInt::getSignedMinValue(BitWidth)));
03604       else if (AllNonPos)
03605         ConservativeResult = ConservativeResult.intersectWith(
03606           ConstantRange(APInt::getSignedMinValue(BitWidth),
03607                         APInt(BitWidth, 1)));
03608     }
03609 
03610     // TODO: non-affine addrec
03611     if (AddRec->isAffine()) {
03612       Type *Ty = AddRec->getType();
03613       const SCEV *MaxBECount = getMaxBackedgeTakenCount(AddRec->getLoop());
03614       if (!isa<SCEVCouldNotCompute>(MaxBECount) &&
03615           getTypeSizeInBits(MaxBECount->getType()) <= BitWidth) {
03616         MaxBECount = getNoopOrZeroExtend(MaxBECount, Ty);
03617 
03618         const SCEV *Start = AddRec->getStart();
03619         const SCEV *Step = AddRec->getStepRecurrence(*this);
03620 
03621         ConstantRange StartRange = getSignedRange(Start);
03622         ConstantRange StepRange = getSignedRange(Step);
03623         ConstantRange MaxBECountRange = getUnsignedRange(MaxBECount);
03624         ConstantRange EndRange =
03625           StartRange.add(MaxBECountRange.multiply(StepRange));
03626 
03627         // Check for overflow. This must be done with ConstantRange arithmetic
03628         // because we could be called from within the ScalarEvolution overflow
03629         // checking code.
03630         ConstantRange ExtStartRange = StartRange.sextOrTrunc(BitWidth*2+1);
03631         ConstantRange ExtStepRange = StepRange.sextOrTrunc(BitWidth*2+1);
03632         ConstantRange ExtMaxBECountRange =
03633           MaxBECountRange.zextOrTrunc(BitWidth*2+1);
03634         ConstantRange ExtEndRange = EndRange.sextOrTrunc(BitWidth*2+1);
03635         if (ExtStartRange.add(ExtMaxBECountRange.multiply(ExtStepRange)) !=
03636             ExtEndRange)
03637           return setSignedRange(AddRec, ConservativeResult);
03638 
03639         APInt Min = APIntOps::smin(StartRange.getSignedMin(),
03640                                    EndRange.getSignedMin());
03641         APInt Max = APIntOps::smax(StartRange.getSignedMax(),
03642                                    EndRange.getSignedMax());
03643         if (Min.isMinSignedValue() && Max.isMaxSignedValue())
03644           return setSignedRange(AddRec, ConservativeResult);
03645         return setSignedRange(AddRec,
03646           ConservativeResult.intersectWith(ConstantRange(Min, Max+1)));
03647       }
03648     }
03649 
03650     return setSignedRange(AddRec, ConservativeResult);
03651   }
03652 
03653   if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
03654     // For a SCEVUnknown, ask ValueTracking.
03655     if (!U->getValue()->getType()->isIntegerTy() && !DL)
03656       return setSignedRange(U, ConservativeResult);
03657     unsigned NS = ComputeNumSignBits(U->getValue(), DL);
03658     if (NS <= 1)
03659       return setSignedRange(U, ConservativeResult);
03660     return setSignedRange(U, ConservativeResult.intersectWith(
03661       ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1),
03662                     APInt::getSignedMaxValue(BitWidth).ashr(NS - 1)+1)));
03663   }
03664 
03665   return setSignedRange(S, ConservativeResult);
03666 }
03667 
03668 /// createSCEV - We know that there is no SCEV for the specified value.
03669 /// Analyze the expression.
03670 ///
03671 const SCEV *ScalarEvolution::createSCEV(Value *V) {
03672   if (!isSCEVable(V->getType()))
03673     return getUnknown(V);
03674 
03675   unsigned Opcode = Instruction::UserOp1;
03676   if (Instruction *I = dyn_cast<Instruction>(V)) {
03677     Opcode = I->getOpcode();
03678 
03679     // Don't attempt to analyze instructions in blocks that aren't
03680     // reachable. Such instructions don't matter, and they aren't required
03681     // to obey basic rules for definitions dominating uses which this
03682     // analysis depends on.
03683     if (!DT->isReachableFromEntry(I->getParent()))
03684       return getUnknown(V);
03685   } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
03686     Opcode = CE->getOpcode();
03687   else if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
03688     return getConstant(CI);
03689   else if (isa<ConstantPointerNull>(V))
03690     return getConstant(V->getType(), 0);
03691   else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V))
03692     return GA->mayBeOverridden() ? getUnknown(V) : getSCEV(GA->getAliasee());
03693   else
03694     return getUnknown(V);
03695 
03696   Operator *U = cast<Operator>(V);
03697   switch (Opcode) {
03698   case Instruction::Add: {
03699     // The simple thing to do would be to just call getSCEV on both operands
03700     // and call getAddExpr with the result. However if we're looking at a
03701     // bunch of things all added together, this can be quite inefficient,
03702     // because it leads to N-1 getAddExpr calls for N ultimate operands.
03703     // Instead, gather up all the operands and make a single getAddExpr call.
03704     // LLVM IR canonical form means we need only traverse the left operands.
03705     //
03706     // Don't apply this instruction's NSW or NUW flags to the new
03707     // expression. The instruction may be guarded by control flow that the
03708     // no-wrap behavior depends on. Non-control-equivalent instructions can be
03709     // mapped to the same SCEV expression, and it would be incorrect to transfer
03710     // NSW/NUW semantics to those operations.
03711     SmallVector<const SCEV *, 4> AddOps;
03712     AddOps.push_back(getSCEV(U->getOperand(1)));
03713     for (Value *Op = U->getOperand(0); ; Op = U->getOperand(0)) {
03714       unsigned Opcode = Op->getValueID() - Value::InstructionVal;
03715       if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
03716         break;
03717       U = cast<Operator>(Op);
03718       const SCEV *Op1 = getSCEV(U->getOperand(1));
03719       if (Opcode == Instruction::Sub)
03720         AddOps.push_back(getNegativeSCEV(Op1));
03721       else
03722         AddOps.push_back(Op1);
03723     }
03724     AddOps.push_back(getSCEV(U->getOperand(0)));
03725     return getAddExpr(AddOps);
03726   }
03727   case Instruction::Mul: {
03728     // Don't transfer NSW/NUW for the same reason as AddExpr.
03729     SmallVector<const SCEV *, 4> MulOps;
03730     MulOps.push_back(getSCEV(U->getOperand(1)));
03731     for (Value *Op = U->getOperand(0);
03732          Op->getValueID() == Instruction::Mul + Value::InstructionVal;
03733          Op = U->getOperand(0)) {
03734       U = cast<Operator>(Op);
03735       MulOps.push_back(getSCEV(U->getOperand(1)));
03736     }
03737     MulOps.push_back(getSCEV(U->getOperand(0)));
03738     return getMulExpr(MulOps);
03739   }
03740   case Instruction::UDiv:
03741     return getUDivExpr(getSCEV(U->getOperand(0)),
03742                        getSCEV(U->getOperand(1)));
03743   case Instruction::Sub:
03744     return getMinusSCEV(getSCEV(U->getOperand(0)),
03745                         getSCEV(U->getOperand(1)));
03746   case Instruction::And:
03747     // For an expression like x&255 that merely masks off the high bits,
03748     // use zext(trunc(x)) as the SCEV expression.
03749     if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
03750       if (CI->isNullValue())
03751         return getSCEV(U->getOperand(1));
03752       if (CI->isAllOnesValue())
03753         return getSCEV(U->getOperand(0));
03754       const APInt &A = CI->getValue();
03755 
03756       // Instcombine's ShrinkDemandedConstant may strip bits out of
03757       // constants, obscuring what would otherwise be a low-bits mask.
03758       // Use ComputeMaskedBits to compute what ShrinkDemandedConstant
03759       // knew about to reconstruct a low-bits mask value.
03760       unsigned LZ = A.countLeadingZeros();
03761       unsigned TZ = A.countTrailingZeros();
03762       unsigned BitWidth = A.getBitWidth();
03763       APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0);
03764       ComputeMaskedBits(U->getOperand(0), KnownZero, KnownOne, DL);
03765 
03766       APInt EffectiveMask =
03767           APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ);
03768       if ((LZ != 0 || TZ != 0) && !((~A & ~KnownZero) & EffectiveMask)) {
03769         const SCEV *MulCount = getConstant(
03770             ConstantInt::get(getContext(), APInt::getOneBitSet(BitWidth, TZ)));
03771         return getMulExpr(
03772             getZeroExtendExpr(
03773                 getTruncateExpr(
03774                     getUDivExactExpr(getSCEV(U->getOperand(0)), MulCount),
03775                     IntegerType::get(getContext(), BitWidth - LZ - TZ)),
03776                 U->getType()),
03777             MulCount);
03778       }
03779     }
03780     break;
03781 
03782   case Instruction::Or:
03783     // If the RHS of the Or is a constant, we may have something like:
03784     // X*4+1 which got turned into X*4|1.  Handle this as an Add so loop
03785     // optimizations will transparently handle this case.
03786     //
03787     // In order for this transformation to be safe, the LHS must be of the
03788     // form X*(2^n) and the Or constant must be less than 2^n.
03789     if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
03790       const SCEV *LHS = getSCEV(U->getOperand(0));
03791       const APInt &CIVal = CI->getValue();
03792       if (GetMinTrailingZeros(LHS) >=
03793           (CIVal.getBitWidth() - CIVal.countLeadingZeros())) {
03794         // Build a plain add SCEV.
03795         const SCEV *S = getAddExpr(LHS, getSCEV(CI));
03796         // If the LHS of the add was an addrec and it has no-wrap flags,
03797         // transfer the no-wrap flags, since an or won't introduce a wrap.
03798         if (const SCEVAddRecExpr *NewAR = dyn_cast<SCEVAddRecExpr>(S)) {
03799           const SCEVAddRecExpr *OldAR = cast<SCEVAddRecExpr>(LHS);
03800           const_cast<SCEVAddRecExpr *>(NewAR)->setNoWrapFlags(
03801             OldAR->getNoWrapFlags());
03802         }
03803         return S;
03804       }
03805     }
03806     break;
03807   case Instruction::Xor:
03808     if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
03809       // If the RHS of the xor is a signbit, then this is just an add.
03810       // Instcombine turns add of signbit into xor as a strength reduction step.
03811       if (CI->getValue().isSignBit())
03812         return getAddExpr(getSCEV(U->getOperand(0)),
03813                           getSCEV(U->getOperand(1)));
03814 
03815       // If the RHS of xor is -1, then this is a not operation.
03816       if (CI->isAllOnesValue())
03817         return getNotSCEV(getSCEV(U->getOperand(0)));
03818 
03819       // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
03820       // This is a variant of the check for xor with -1, and it handles
03821       // the case where instcombine has trimmed non-demanded bits out
03822       // of an xor with -1.
03823       if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U->getOperand(0)))
03824         if (ConstantInt *LCI = dyn_cast<ConstantInt>(BO->getOperand(1)))
03825           if (BO->getOpcode() == Instruction::And &&
03826               LCI->getValue() == CI->getValue())
03827             if (const SCEVZeroExtendExpr *Z =
03828                   dyn_cast<SCEVZeroExtendExpr>(getSCEV(U->getOperand(0)))) {
03829               Type *UTy = U->getType();
03830               const SCEV *Z0 = Z->getOperand();
03831               Type *Z0Ty = Z0->getType();
03832               unsigned Z0TySize = getTypeSizeInBits(Z0Ty);
03833 
03834               // If C is a low-bits mask, the zero extend is serving to
03835               // mask off the high bits. Complement the operand and
03836               // re-apply the zext.
03837               if (APIntOps::isMask(Z0TySize, CI->getValue()))
03838                 return getZeroExtendExpr(getNotSCEV(Z0), UTy);
03839 
03840               // If C is a single bit, it may be in the sign-bit position
03841               // before the zero-extend. In this case, represent the xor
03842               // using an add, which is equivalent, and re-apply the zext.
03843               APInt Trunc = CI->getValue().trunc(Z0TySize);
03844               if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() &&
03845                   Trunc.isSignBit())
03846                 return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)),
03847                                          UTy);
03848             }
03849     }
03850     break;
03851 
03852   case Instruction::Shl:
03853     // Turn shift left of a constant amount into a multiply.
03854     if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) {
03855       uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth();
03856 
03857       // If the shift count is not less than the bitwidth, the result of
03858       // the shift is undefined. Don't try to analyze it, because the
03859       // resolution chosen here may differ from the resolution chosen in
03860       // other parts of the compiler.
03861       if (SA->getValue().uge(BitWidth))
03862         break;
03863 
03864       Constant *X = ConstantInt::get(getContext(),
03865         APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
03866       return getMulExpr(getSCEV(U->getOperand(0)), getSCEV(X));
03867     }
03868     break;
03869 
03870   case Instruction::LShr:
03871     // Turn logical shift right of a constant into a unsigned divide.
03872     if (ConstantInt *SA = dyn_cast<ConstantInt>(U->getOperand(1))) {
03873       uint32_t BitWidth = cast<IntegerType>(U->getType())->getBitWidth();
03874 
03875       // If the shift count is not less than the bitwidth, the result of
03876       // the shift is undefined. Don't try to analyze it, because the
03877       // resolution chosen here may differ from the resolution chosen in
03878       // other parts of the compiler.
03879       if (SA->getValue().uge(BitWidth))
03880         break;
03881 
03882       Constant *X = ConstantInt::get(getContext(),
03883         APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
03884       return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(X));
03885     }
03886     break;
03887 
03888   case Instruction::AShr:
03889     // For a two-shift sext-inreg, use sext(trunc(x)) as the SCEV expression.
03890     if (ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1)))
03891       if (Operator *L = dyn_cast<Operator>(U->getOperand(0)))
03892         if (L->getOpcode() == Instruction::Shl &&
03893             L->getOperand(1) == U->getOperand(1)) {
03894           uint64_t BitWidth = getTypeSizeInBits(U->getType());
03895 
03896           // If the shift count is not less than the bitwidth, the result of
03897           // the shift is undefined. Don't try to analyze it, because the
03898           // resolution chosen here may differ from the resolution chosen in
03899           // other parts of the compiler.
03900           if (CI->getValue().uge(BitWidth))
03901             break;
03902 
03903           uint64_t Amt = BitWidth - CI->getZExtValue();
03904           if (Amt == BitWidth)
03905             return getSCEV(L->getOperand(0));       // shift by zero --> noop
03906           return
03907             getSignExtendExpr(getTruncateExpr(getSCEV(L->getOperand(0)),
03908                                               IntegerType::get(getContext(),
03909                                                                Amt)),
03910                               U->getType());
03911         }
03912     break;
03913 
03914   case Instruction::Trunc:
03915     return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType());
03916 
03917   case Instruction::ZExt:
03918     return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType());
03919 
03920   case Instruction::SExt:
03921     return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType());
03922 
03923   case Instruction::BitCast:
03924     // BitCasts are no-op casts so we just eliminate the cast.
03925     if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType()))
03926       return getSCEV(U->getOperand(0));
03927     break;
03928 
03929   // It's tempting to handle inttoptr and ptrtoint as no-ops, however this can
03930   // lead to pointer expressions which cannot safely be expanded to GEPs,
03931   // because ScalarEvolution doesn't respect the GEP aliasing rules when
03932   // simplifying integer expressions.
03933 
03934   case Instruction::GetElementPtr:
03935     return createNodeForGEP(cast<GEPOperator>(U));
03936 
03937   case Instruction::PHI:
03938     return createNodeForPHI(cast<PHINode>(U));
03939 
03940   case Instruction::Select:
03941     // This could be a smax or umax that was lowered earlier.
03942     // Try to recover it.
03943     if (ICmpInst *ICI = dyn_cast<ICmpInst>(U->getOperand(0))) {
03944       Value *LHS = ICI->getOperand(0);
03945       Value *RHS = ICI->getOperand(1);
03946       switch (ICI->getPredicate()) {
03947       case ICmpInst::ICMP_SLT:
03948       case ICmpInst::ICMP_SLE:
03949         std::swap(LHS, RHS);
03950         // fall through
03951       case ICmpInst::ICMP_SGT:
03952       case ICmpInst::ICMP_SGE:
03953         // a >s b ? a+x : b+x  ->  smax(a, b)+x
03954         // a >s b ? b+x : a+x  ->  smin(a, b)+x
03955         if (LHS->getType() == U->getType()) {
03956           const SCEV *LS = getSCEV(LHS);
03957           const SCEV *RS = getSCEV(RHS);
03958           const SCEV *LA = getSCEV(U->getOperand(1));
03959           const SCEV *RA = getSCEV(U->getOperand(2));
03960           const SCEV *LDiff = getMinusSCEV(LA, LS);
03961           const SCEV *RDiff = getMinusSCEV(RA, RS);
03962           if (LDiff == RDiff)
03963             return getAddExpr(getSMaxExpr(LS, RS), LDiff);
03964           LDiff = getMinusSCEV(LA, RS);
03965           RDiff = getMinusSCEV(RA, LS);
03966           if (LDiff == RDiff)
03967             return getAddExpr(getSMinExpr(LS, RS), LDiff);
03968         }
03969         break;
03970       case ICmpInst::ICMP_ULT:
03971       case ICmpInst::ICMP_ULE:
03972         std::swap(LHS, RHS);
03973         // fall through
03974       case ICmpInst::ICMP_UGT:
03975       case ICmpInst::ICMP_UGE:
03976         // a >u b ? a+x : b+x  ->  umax(a, b)+x
03977         // a >u b ? b+x : a+x  ->  umin(a, b)+x
03978         if (LHS->getType() == U->getType()) {
03979           const SCEV *LS = getSCEV(LHS);
03980           const SCEV *RS = getSCEV(RHS);
03981           const SCEV *LA = getSCEV(U->getOperand(1));
03982           const SCEV *RA = getSCEV(U->getOperand(2));
03983           const SCEV *LDiff = getMinusSCEV(LA, LS);
03984           const SCEV *RDiff = getMinusSCEV(RA, RS);
03985           if (LDiff == RDiff)
03986             return getAddExpr(getUMaxExpr(LS, RS), LDiff);
03987           LDiff = getMinusSCEV(LA, RS);
03988           RDiff = getMinusSCEV(RA, LS);
03989           if (LDiff == RDiff)
03990             return getAddExpr(getUMinExpr(LS, RS), LDiff);
03991         }
03992         break;
03993       case ICmpInst::ICMP_NE:
03994         // n != 0 ? n+x : 1+x  ->  umax(n, 1)+x
03995         if (LHS->getType() == U->getType() &&
03996             isa<ConstantInt>(RHS) &&
03997             cast<ConstantInt>(RHS)->isZero()) {
03998           const SCEV *One = getConstant(LHS->getType(), 1);
03999           const SCEV *LS = getSCEV(LHS);
04000           const SCEV *LA = getSCEV(U->getOperand(1));
04001           const SCEV *RA = getSCEV(U->getOperand(2));
04002           const SCEV *LDiff = getMinusSCEV(LA, LS);
04003           const SCEV *RDiff = getMinusSCEV(RA, One);
04004           if (LDiff == RDiff)
04005             return getAddExpr(getUMaxExpr(One, LS), LDiff);
04006         }
04007         break;
04008       case ICmpInst::ICMP_EQ:
04009         // n == 0 ? 1+x : n+x  ->  umax(n, 1)+x
04010         if (LHS->getType() == U->getType() &&
04011             isa<ConstantInt>(RHS) &&
04012             cast<ConstantInt>(RHS)->isZero()) {
04013           const SCEV *One = getConstant(LHS->getType(), 1);
04014           const SCEV *LS = getSCEV(LHS);
04015           const SCEV *LA = getSCEV(U->getOperand(1));
04016           const SCEV *RA = getSCEV(U->getOperand(2));
04017           const SCEV *LDiff = getMinusSCEV(LA, One);
04018           const SCEV *RDiff = getMinusSCEV(RA, LS);
04019           if (LDiff == RDiff)
04020             return getAddExpr(getUMaxExpr(One, LS), LDiff);
04021         }
04022         break;
04023       default:
04024         break;
04025       }
04026     }
04027 
04028   default: // We cannot analyze this expression.
04029     break;
04030   }
04031 
04032   return getUnknown(V);
04033 }
04034 
04035 
04036 
04037 //===----------------------------------------------------------------------===//
04038 //                   Iteration Count Computation Code
04039 //
04040 
04041 /// getSmallConstantTripCount - Returns the maximum trip count of this loop as a
04042 /// normal unsigned value. Returns 0 if the trip count is unknown or not
04043 /// constant. Will also return 0 if the maximum trip count is very large (>=
04044 /// 2^32).
04045 ///
04046 /// This "trip count" assumes that control exits via ExitingBlock. More
04047 /// precisely, it is the number of times that control may reach ExitingBlock
04048 /// before taking the branch. For loops with multiple exits, it may not be the
04049 /// number times that the loop header executes because the loop may exit
04050 /// prematurely via another branch.
04051 ///
04052 /// FIXME: We conservatively call getBackedgeTakenCount(L) instead of
04053 /// getExitCount(L, ExitingBlock) to compute a safe trip count considering all
04054 /// loop exits. getExitCount() may return an exact count for this branch
04055 /// assuming no-signed-wrap. The number of well-defined iterations may actually
04056 /// be higher than this trip count if this exit test is skipped and the loop
04057 /// exits via a different branch. Ideally, getExitCount() would know whether it
04058 /// depends on a NSW assumption, and we would only fall back to a conservative
04059 /// trip count in that case.
04060 unsigned ScalarEvolution::
04061 getSmallConstantTripCount(Loop *L, BasicBlock * /*ExitingBlock*/) {
04062   const SCEVConstant *ExitCount =
04063     dyn_cast<SCEVConstant>(getBackedgeTakenCount(L));
04064   if (!ExitCount)
04065     return 0;
04066 
04067   ConstantInt *ExitConst = ExitCount->getValue();
04068 
04069   // Guard against huge trip counts.
04070   if (ExitConst->getValue().getActiveBits() > 32)
04071     return 0;
04072 
04073   // In case of integer overflow, this returns 0, which is correct.
04074   return ((unsigned)ExitConst->getZExtValue()) + 1;
04075 }
04076 
04077 /// getSmallConstantTripMultiple - Returns the largest constant divisor of the
04078 /// trip count of this loop as a normal unsigned value, if possible. This
04079 /// means that the actual trip count is always a multiple of the returned
04080 /// value (don't forget the trip count could very well be zero as well!).
04081 ///
04082 /// Returns 1 if the trip count is unknown or not guaranteed to be the
04083 /// multiple of a constant (which is also the case if the trip count is simply
04084 /// constant, use getSmallConstantTripCount for that case), Will also return 1
04085 /// if the trip count is very large (>= 2^32).
04086 ///
04087 /// As explained in the comments for getSmallConstantTripCount, this assumes
04088 /// that control exits the loop via ExitingBlock.
04089 unsigned ScalarEvolution::
04090 getSmallConstantTripMultiple(Loop *L, BasicBlock * /*ExitingBlock*/) {
04091   const SCEV *ExitCount = getBackedgeTakenCount(L);
04092   if (ExitCount == getCouldNotCompute())
04093     return 1;
04094 
04095   // Get the trip count from the BE count by adding 1.
04096   const SCEV *TCMul = getAddExpr(ExitCount,
04097                                  getConstant(ExitCount->getType(), 1));
04098   // FIXME: SCEV distributes multiplication as V1*C1 + V2*C1. We could attempt
04099   // to factor simple cases.
04100   if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(TCMul))
04101     TCMul = Mul->getOperand(0);
04102 
04103   const SCEVConstant *MulC = dyn_cast<SCEVConstant>(TCMul);
04104   if (!MulC)
04105     return 1;
04106 
04107   ConstantInt *Result = MulC->getValue();
04108 
04109   // Guard against huge trip counts (this requires checking
04110   // for zero to handle the case where the trip count == -1 and the
04111   // addition wraps).
04112   if (!Result || Result->getValue().getActiveBits() > 32 ||
04113       Result->getValue().getActiveBits() == 0)
04114     return 1;
04115 
04116   return (unsigned)Result->getZExtValue();
04117 }
04118 
04119 // getExitCount - Get the expression for the number of loop iterations for which
04120 // this loop is guaranteed not to exit via ExitingBlock. Otherwise return
04121 // SCEVCouldNotCompute.
04122 const SCEV *ScalarEvolution::getExitCount(Loop *L, BasicBlock *ExitingBlock) {
04123   return getBackedgeTakenInfo(L).getExact(ExitingBlock, this);
04124 }
04125 
04126 /// getBackedgeTakenCount - If the specified loop has a predictable
04127 /// backedge-taken count, return it, otherwise return a SCEVCouldNotCompute
04128 /// object. The backedge-taken count is the number of times the loop header
04129 /// will be branched to from within the loop. This is one less than the
04130 /// trip count of the loop, since it doesn't count the first iteration,
04131 /// when the header is branched to from outside the loop.
04132 ///
04133 /// Note that it is not valid to call this method on a loop without a
04134 /// loop-invariant backedge-taken count (see
04135 /// hasLoopInvariantBackedgeTakenCount).
04136 ///
04137 const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L) {
04138   return getBackedgeTakenInfo(L).getExact(this);
04139 }
04140 
04141 /// getMaxBackedgeTakenCount - Similar to getBackedgeTakenCount, except
04142 /// return the least SCEV value that is known never to be less than the
04143 /// actual backedge taken count.
04144 const SCEV *ScalarEvolution::getMaxBackedgeTakenCount(const Loop *L) {
04145   return getBackedgeTakenInfo(L).getMax(this);
04146 }
04147 
04148 /// PushLoopPHIs - Push PHI nodes in the header of the given loop
04149 /// onto the given Worklist.
04150 static void
04151 PushLoopPHIs(const Loop *L, SmallVectorImpl<Instruction *> &Worklist) {
04152   BasicBlock *Header = L->getHeader();
04153 
04154   // Push all Loop-header PHIs onto the Worklist stack.
04155   for (BasicBlock::iterator I = Header->begin();
04156        PHINode *PN = dyn_cast<PHINode>(I); ++I)
04157     Worklist.push_back(PN);
04158 }
04159 
04160 const ScalarEvolution::BackedgeTakenInfo &
04161 ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
04162   // Initially insert an invalid entry for this loop. If the insertion
04163   // succeeds, proceed to actually compute a backedge-taken count and
04164   // update the value. The temporary CouldNotCompute value tells SCEV
04165   // code elsewhere that it shouldn't attempt to request a new
04166   // backedge-taken count, which could result in infinite recursion.
04167   std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
04168     BackedgeTakenCounts.insert(std::make_pair(L, BackedgeTakenInfo()));
04169   if (!Pair.second)
04170     return Pair.first->second;
04171 
04172   // ComputeBackedgeTakenCount may allocate memory for its result. Inserting it
04173   // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
04174   // must be cleared in this scope.
04175   BackedgeTakenInfo Result = ComputeBackedgeTakenCount(L);
04176 
04177   if (Result.getExact(this) != getCouldNotCompute()) {
04178     assert(isLoopInvariant(Result.getExact(this), L) &&
04179            isLoopInvariant(Result.getMax(this), L) &&
04180            "Computed backedge-taken count isn't loop invariant for loop!");
04181     ++NumTripCountsComputed;
04182   }
04183   else if (Result.getMax(this) == getCouldNotCompute() &&
04184            isa<PHINode>(L->getHeader()->begin())) {
04185     // Only count loops that have phi nodes as not being computable.
04186     ++NumTripCountsNotComputed;
04187   }
04188 
04189   // Now that we know more about the trip count for this loop, forget any
04190   // existing SCEV values for PHI nodes in this loop since they are only
04191   // conservative estimates made without the benefit of trip count
04192   // information. This is similar to the code in forgetLoop, except that
04193   // it handles SCEVUnknown PHI nodes specially.
04194   if (Result.hasAnyInfo()) {
04195     SmallVector<Instruction *, 16> Worklist;
04196     PushLoopPHIs(L, Worklist);
04197 
04198     SmallPtrSet<Instruction *, 8> Visited;
04199     while (!Worklist.empty()) {
04200       Instruction *I = Worklist.pop_back_val();
04201       if (!Visited.insert(I)) continue;
04202 
04203       ValueExprMapType::iterator It =
04204         ValueExprMap.find_as(static_cast<Value *>(I));
04205       if (It != ValueExprMap.end()) {
04206         const SCEV *Old = It->second;
04207 
04208         // SCEVUnknown for a PHI either means that it has an unrecognized
04209         // structure, or it's a PHI that's in the progress of being computed
04210         // by createNodeForPHI.  In the former case, additional loop trip
04211         // count information isn't going to change anything. In the later
04212         // case, createNodeForPHI will perform the necessary updates on its
04213         // own when it gets to that point.
04214         if (!isa<PHINode>(I) || !isa<SCEVUnknown>(Old)) {
04215           forgetMemoizedResults(Old);
04216           ValueExprMap.erase(It);
04217         }
04218         if (PHINode *PN = dyn_cast<PHINode>(I))
04219           ConstantEvolutionLoopExitValue.erase(PN);
04220       }
04221 
04222       PushDefUseChildren(I, Worklist);
04223     }
04224   }
04225 
04226   // Re-lookup the insert position, since the call to
04227   // ComputeBackedgeTakenCount above could result in a
04228   // recusive call to getBackedgeTakenInfo (on a different
04229   // loop), which would invalidate the iterator computed
04230   // earlier.
04231   return BackedgeTakenCounts.find(L)->second = Result;
04232 }
04233 
04234 /// forgetLoop - This method should be called by the client when it has
04235 /// changed a loop in a way that may effect ScalarEvolution's ability to
04236 /// compute a trip count, or if the loop is deleted.
04237 void ScalarEvolution::forgetLoop(const Loop *L) {
04238   // Drop any stored trip count value.
04239   DenseMap<const Loop*, BackedgeTakenInfo>::iterator BTCPos =
04240     BackedgeTakenCounts.find(L);
04241   if (BTCPos != BackedgeTakenCounts.end()) {
04242     BTCPos->second.clear();
04243     BackedgeTakenCounts.erase(BTCPos);
04244   }
04245 
04246   // Drop information about expressions based on loop-header PHIs.
04247   SmallVector<Instruction *, 16> Worklist;
04248   PushLoopPHIs(L, Worklist);
04249 
04250   SmallPtrSet<Instruction *, 8> Visited;
04251   while (!Worklist.empty()) {
04252     Instruction *I = Worklist.pop_back_val();
04253     if (!Visited.insert(I)) continue;
04254 
04255     ValueExprMapType::iterator It =
04256       ValueExprMap.find_as(static_cast<Value *>(I));
04257     if (It != ValueExprMap.end()) {
04258       forgetMemoizedResults(It->second);
04259       ValueExprMap.erase(It);
04260       if (PHINode *PN = dyn_cast<PHINode>(I))
04261         ConstantEvolutionLoopExitValue.erase(PN);
04262     }
04263 
04264     PushDefUseChildren(I, Worklist);
04265   }
04266 
04267   // Forget all contained loops too, to avoid dangling entries in the
04268   // ValuesAtScopes map.
04269   for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I)
04270     forgetLoop(*I);
04271 }
04272 
04273 /// forgetValue - This method should be called by the client when it has
04274 /// changed a value in a way that may effect its value, or which may
04275 /// disconnect it from a def-use chain linking it to a loop.
04276 void ScalarEvolution::forgetValue(Value *V) {
04277   Instruction *I = dyn_cast<Instruction>(V);
04278   if (!I) return;
04279 
04280   // Drop information about expressions based on loop-header PHIs.
04281   SmallVector<Instruction *, 16> Worklist;
04282   Worklist.push_back(I);
04283 
04284   SmallPtrSet<Instruction *, 8> Visited;
04285   while (!Worklist.empty()) {
04286     I = Worklist.pop_back_val();
04287     if (!Visited.insert(I)) continue;
04288 
04289     ValueExprMapType::iterator It =
04290       ValueExprMap.find_as(static_cast<Value *>(I));
04291     if (It != ValueExprMap.end()) {
04292       forgetMemoizedResults(It->second);
04293       ValueExprMap.erase(It);
04294       if (PHINode *PN = dyn_cast<PHINode>(I))
04295         ConstantEvolutionLoopExitValue.erase(PN);
04296     }
04297 
04298     PushDefUseChildren(I, Worklist);
04299   }
04300 }
04301 
04302 /// getExact - Get the exact loop backedge taken count considering all loop
04303 /// exits. A computable result can only be return for loops with a single exit.
04304 /// Returning the minimum taken count among all exits is incorrect because one
04305 /// of the loop's exit limit's may have been skipped. HowFarToZero assumes that
04306 /// the limit of each loop test is never skipped. This is a valid assumption as
04307 /// long as the loop exits via that test. For precise results, it is the
04308 /// caller's responsibility to specify the relevant loop exit using
04309 /// getExact(ExitingBlock, SE).
04310 const SCEV *
04311 ScalarEvolution::BackedgeTakenInfo::getExact(ScalarEvolution *SE) const {
04312   // If any exits were not computable, the loop is not computable.
04313   if (!ExitNotTaken.isCompleteList()) return SE->getCouldNotCompute();
04314 
04315   // We need exactly one computable exit.
04316   if (!ExitNotTaken.ExitingBlock) return SE->getCouldNotCompute();
04317   assert(ExitNotTaken.ExactNotTaken && "uninitialized not-taken info");
04318 
04319   const SCEV *BECount = nullptr;
04320   for (const ExitNotTakenInfo *ENT = &ExitNotTaken;
04321        ENT != nullptr; ENT = ENT->getNextExit()) {
04322 
04323     assert(ENT->ExactNotTaken != SE->getCouldNotCompute() && "bad exit SCEV");
04324 
04325     if (!BECount)
04326       BECount = ENT->ExactNotTaken;
04327     else if (BECount != ENT->ExactNotTaken)
04328       return SE->getCouldNotCompute();
04329   }
04330   assert(BECount && "Invalid not taken count for loop exit");
04331   return BECount;
04332 }
04333 
04334 /// getExact - Get the exact not taken count for this loop exit.
04335 const SCEV *
04336 ScalarEvolution::BackedgeTakenInfo::getExact(BasicBlock *ExitingBlock,
04337                                              ScalarEvolution *SE) const {
04338   for (const ExitNotTakenInfo *ENT = &ExitNotTaken;
04339        ENT != nullptr; ENT = ENT->getNextExit()) {
04340 
04341     if (ENT->ExitingBlock == ExitingBlock)
04342       return ENT->ExactNotTaken;
04343   }
04344   return SE->getCouldNotCompute();
04345 }
04346 
04347 /// getMax - Get the max backedge taken count for the loop.
04348 const SCEV *
04349 ScalarEvolution::BackedgeTakenInfo::getMax(ScalarEvolution *SE) const {
04350   return Max ? Max : SE->getCouldNotCompute();
04351 }
04352 
04353 bool ScalarEvolution::BackedgeTakenInfo::hasOperand(const SCEV *S,
04354                                                     ScalarEvolution *SE) const {
04355   if (Max && Max != SE->getCouldNotCompute() && SE->hasOperand(Max, S))
04356     return true;
04357 
04358   if (!ExitNotTaken.ExitingBlock)
04359     return false;
04360 
04361   for (const ExitNotTakenInfo *ENT = &ExitNotTaken;
04362        ENT != nullptr; ENT = ENT->getNextExit()) {
04363 
04364     if (ENT->ExactNotTaken != SE->getCouldNotCompute()
04365         && SE->hasOperand(ENT->ExactNotTaken, S)) {
04366       return true;
04367     }
04368   }
04369   return false;
04370 }
04371 
04372 /// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
04373 /// computable exit into a persistent ExitNotTakenInfo array.
04374 ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
04375   SmallVectorImpl< std::pair<BasicBlock *, const SCEV *> > &ExitCounts,
04376   bool Complete, const SCEV *MaxCount) : Max(MaxCount) {
04377 
04378   if (!Complete)
04379     ExitNotTaken.setIncomplete();
04380 
04381   unsigned NumExits = ExitCounts.size();
04382   if (NumExits == 0) return;
04383 
04384   ExitNotTaken.ExitingBlock = ExitCounts[0].first;
04385   ExitNotTaken.ExactNotTaken = ExitCounts[0].second;
04386   if (NumExits == 1) return;
04387 
04388   // Handle the rare case of multiple computable exits.
04389   ExitNotTakenInfo *ENT = new ExitNotTakenInfo[NumExits-1];
04390 
04391   ExitNotTakenInfo *PrevENT = &ExitNotTaken;
04392   for (unsigned i = 1; i < NumExits; ++i, PrevENT = ENT, ++ENT) {
04393     PrevENT->setNextExit(ENT);
04394     ENT->ExitingBlock = ExitCounts[i].first;
04395     ENT->ExactNotTaken = ExitCounts[i].second;
04396   }
04397 }
04398 
04399 /// clear - Invalidate this result and free the ExitNotTakenInfo array.
04400 void ScalarEvolution::BackedgeTakenInfo::clear() {
04401   ExitNotTaken.ExitingBlock = nullptr;
04402   ExitNotTaken.ExactNotTaken = nullptr;
04403   delete[] ExitNotTaken.getNextExit();
04404 }
04405 
04406 /// ComputeBackedgeTakenCount - Compute the number of times the backedge
04407 /// of the specified loop will execute.
04408 ScalarEvolution::BackedgeTakenInfo
04409 ScalarEvolution::ComputeBackedgeTakenCount(const Loop *L) {
04410   SmallVector<BasicBlock *, 8> ExitingBlocks;
04411   L->getExitingBlocks(ExitingBlocks);
04412 
04413   // Examine all exits and pick the most conservative values.
04414   const SCEV *MaxBECount = getCouldNotCompute();
04415   bool CouldComputeBECount = true;
04416   BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
04417   const SCEV *LatchMaxCount = nullptr;
04418   SmallVector<std::pair<BasicBlock *, const SCEV *>, 4> ExitCounts;
04419   for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) {
04420     ExitLimit EL = ComputeExitLimit(L, ExitingBlocks[i]);
04421     if (EL.Exact == getCouldNotCompute())
04422       // We couldn't compute an exact value for this exit, so
04423       // we won't be able to compute an exact value for the loop.
04424       CouldComputeBECount = false;
04425     else
04426       ExitCounts.push_back(std::make_pair(ExitingBlocks[i], EL.Exact));
04427 
04428     if (MaxBECount == getCouldNotCompute())
04429       MaxBECount = EL.Max;
04430     else if (EL.Max != getCouldNotCompute()) {
04431       // We cannot take the "min" MaxBECount, because non-unit stride loops may
04432       // skip some loop tests. Taking the max over the exits is sufficiently
04433       // conservative.  TODO: We could do better taking into consideration
04434       // non-latch exits that dominate the latch.
04435       if (EL.MustExit && ExitingBlocks[i] == Latch)
04436         LatchMaxCount = EL.Max;
04437       else
04438         MaxBECount = getUMaxFromMismatchedTypes(MaxBECount, EL.Max);
04439     }
04440   }
04441   // Be more precise in the easy case of a loop latch that must exit.
04442   if (LatchMaxCount) {
04443     MaxBECount = getUMinFromMismatchedTypes(MaxBECount, LatchMaxCount);
04444   }
04445   return BackedgeTakenInfo(ExitCounts, CouldComputeBECount, MaxBECount);
04446 }
04447 
04448 /// ComputeExitLimit - Compute the number of times the backedge of the specified
04449 /// loop will execute if it exits via the specified block.
04450 ScalarEvolution::ExitLimit
04451 ScalarEvolution::ComputeExitLimit(const Loop *L, BasicBlock *ExitingBlock) {
04452 
04453   // Okay, we've chosen an exiting block.  See what condition causes us to
04454   // exit at this block and remember the exit block and whether all other targets
04455   // lead to the loop header.
04456   bool MustExecuteLoopHeader = true;
04457   BasicBlock *Exit = nullptr;
04458   for (succ_iterator SI = succ_begin(ExitingBlock), SE = succ_end(ExitingBlock);
04459        SI != SE; ++SI)
04460     if (!L->contains(*SI)) {
04461       if (Exit) // Multiple exit successors.
04462         return getCouldNotCompute();
04463       Exit = *SI;
04464     } else if (*SI != L->getHeader()) {
04465       MustExecuteLoopHeader = false;
04466     }
04467 
04468   // At this point, we know we have a conditional branch that determines whether
04469   // the loop is exited.  However, we don't know if the branch is executed each
04470   // time through the loop.  If not, then the execution count of the branch will
04471   // not be equal to the trip count of the loop.
04472   //
04473   // Currently we check for this by checking to see if the Exit branch goes to
04474   // the loop header.  If so, we know it will always execute the same number of
04475   // times as the loop.  We also handle the case where the exit block *is* the
04476   // loop header.  This is common for un-rotated loops.
04477   //
04478   // If both of those tests fail, walk up the unique predecessor chain to the
04479   // header, stopping if there is an edge that doesn't exit the loop. If the
04480   // header is reached, the execution count of the branch will be equal to the
04481   // trip count of the loop.
04482   //
04483   //  More extensive analysis could be done to handle more cases here.
04484   //
04485   if (!MustExecuteLoopHeader && ExitingBlock != L->getHeader()) {
04486     // The simple checks failed, try climbing the unique predecessor chain
04487     // up to the header.
04488     bool Ok = false;
04489     for (BasicBlock *BB = ExitingBlock; BB; ) {
04490       BasicBlock *Pred = BB->getUniquePredecessor();
04491       if (!Pred)
04492         return getCouldNotCompute();
04493       TerminatorInst *PredTerm = Pred->getTerminator();
04494       for (unsigned i = 0, e = PredTerm->getNumSuccessors(); i != e; ++i) {
04495         BasicBlock *PredSucc = PredTerm->getSuccessor(i);
04496         if (PredSucc == BB)
04497           continue;
04498         // If the predecessor has a successor that isn't BB and isn't
04499         // outside the loop, assume the worst.
04500         if (L->contains(PredSucc))
04501           return getCouldNotCompute();
04502       }
04503       if (Pred == L->getHeader()) {
04504         Ok = true;
04505         break;
04506       }
04507       BB = Pred;
04508     }
04509     if (!Ok)
04510       return getCouldNotCompute();
04511   }
04512 
04513   TerminatorInst *Term = ExitingBlock->getTerminator();
04514   if (BranchInst *BI = dyn_cast<BranchInst>(Term)) {
04515     assert(BI->isConditional() && "If unconditional, it can't be in loop!");
04516     // Proceed to the next level to examine the exit condition expression.
04517     return ComputeExitLimitFromCond(L, BI->getCondition(), BI->getSuccessor(0),
04518                                     BI->getSuccessor(1),
04519                                     /*IsSubExpr=*/false);
04520   }
04521 
04522   if (SwitchInst *SI = dyn_cast<SwitchInst>(Term))
04523     return ComputeExitLimitFromSingleExitSwitch(L, SI, Exit,
04524                                                 /*IsSubExpr=*/false);
04525 
04526   return getCouldNotCompute();
04527 }
04528 
04529 /// ComputeExitLimitFromCond - Compute the number of times the
04530 /// backedge of the specified loop will execute if its exit condition
04531 /// were a conditional branch of ExitCond, TBB, and FBB.
04532 ///
04533 /// @param IsSubExpr is true if ExitCond does not directly control the exit
04534 /// branch. In this case, we cannot assume that the loop only exits when the
04535 /// condition is true and cannot infer that failing to meet the condition prior
04536 /// to integer wraparound results in undefined behavior.
04537 ScalarEvolution::ExitLimit
04538 ScalarEvolution::ComputeExitLimitFromCond(const Loop *L,
04539                                           Value *ExitCond,
04540                                           BasicBlock *TBB,
04541                                           BasicBlock *FBB,
04542                                           bool IsSubExpr) {
04543   // Check if the controlling expression for this loop is an And or Or.
04544   if (BinaryOperator *BO = dyn_cast<BinaryOperator>(ExitCond)) {
04545     if (BO->getOpcode() == Instruction::And) {
04546       // Recurse on the operands of the and.
04547       bool EitherMayExit = L->contains(TBB);
04548       ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB,
04549                                                IsSubExpr || EitherMayExit);
04550       ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB,
04551                                                IsSubExpr || EitherMayExit);
04552       const SCEV *BECount = getCouldNotCompute();
04553       const SCEV *MaxBECount = getCouldNotCompute();
04554       bool MustExit = false;
04555       if (EitherMayExit) {
04556         // Both conditions must be true for the loop to continue executing.
04557         // Choose the less conservative count.
04558         if (EL0.Exact == getCouldNotCompute() ||
04559             EL1.Exact == getCouldNotCompute())
04560           BECount = getCouldNotCompute();
04561         else
04562           BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact);
04563         if (EL0.Max == getCouldNotCompute())
04564           MaxBECount = EL1.Max;
04565         else if (EL1.Max == getCouldNotCompute())
04566           MaxBECount = EL0.Max;
04567         else
04568           MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max);
04569         MustExit = EL0.MustExit || EL1.MustExit;
04570       } else {
04571         // Both conditions must be true at the same time for the loop to exit.
04572         // For now, be conservative.
04573         assert(L->contains(FBB) && "Loop block has no successor in loop!");
04574         if (EL0.Max == EL1.Max)
04575           MaxBECount = EL0.Max;
04576         if (EL0.Exact == EL1.Exact)
04577           BECount = EL0.Exact;
04578         MustExit = EL0.MustExit && EL1.MustExit;
04579       }
04580 
04581       return ExitLimit(BECount, MaxBECount, MustExit);
04582     }
04583     if (BO->getOpcode() == Instruction::Or) {
04584       // Recurse on the operands of the or.
04585       bool EitherMayExit = L->contains(FBB);
04586       ExitLimit EL0 = ComputeExitLimitFromCond(L, BO->getOperand(0), TBB, FBB,
04587                                                IsSubExpr || EitherMayExit);
04588       ExitLimit EL1 = ComputeExitLimitFromCond(L, BO->getOperand(1), TBB, FBB,
04589                                                IsSubExpr || EitherMayExit);
04590       const SCEV *BECount = getCouldNotCompute();
04591       const SCEV *MaxBECount = getCouldNotCompute();
04592       bool MustExit = false;
04593       if (EitherMayExit) {
04594         // Both conditions must be false for the loop to continue executing.
04595         // Choose the less conservative count.
04596         if (EL0.Exact == getCouldNotCompute() ||
04597             EL1.Exact == getCouldNotCompute())
04598           BECount = getCouldNotCompute();
04599         else
04600           BECount = getUMinFromMismatchedTypes(EL0.Exact, EL1.Exact);
04601         if (EL0.Max == getCouldNotCompute())
04602           MaxBECount = EL1.Max;
04603         else if (EL1.Max == getCouldNotCompute())
04604           MaxBECount = EL0.Max;
04605         else
04606           MaxBECount = getUMinFromMismatchedTypes(EL0.Max, EL1.Max);
04607         MustExit = EL0.MustExit || EL1.MustExit;
04608       } else {
04609         // Both conditions must be false at the same time for the loop to exit.
04610         // For now, be conservative.
04611         assert(L->contains(TBB) && "Loop block has no successor in loop!");
04612         if (EL0.Max == EL1.Max)
04613           MaxBECount = EL0.Max;
04614         if (EL0.Exact == EL1.Exact)
04615           BECount = EL0.Exact;
04616         MustExit = EL0.MustExit && EL1.MustExit;
04617       }
04618 
04619       return ExitLimit(BECount, MaxBECount, MustExit);
04620     }
04621   }
04622 
04623   // With an icmp, it may be feasible to compute an exact backedge-taken count.
04624   // Proceed to the next level to examine the icmp.
04625   if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(ExitCond))
04626     return ComputeExitLimitFromICmp(L, ExitCondICmp, TBB, FBB, IsSubExpr);
04627 
04628   // Check for a constant condition. These are normally stripped out by
04629   // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
04630   // preserve the CFG and is temporarily leaving constant conditions
04631   // in place.
04632   if (ConstantInt *CI = dyn_cast<ConstantInt>(ExitCond)) {
04633     if (L->contains(FBB) == !CI->getZExtValue())
04634       // The backedge is always taken.
04635       return getCouldNotCompute();
04636     else
04637       // The backedge is never taken.
04638       return getConstant(CI->getType(), 0);
04639   }
04640 
04641   // If it's not an integer or pointer comparison then compute it the hard way.
04642   return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB));
04643 }
04644 
04645 /// ComputeExitLimitFromICmp - Compute the number of times the
04646 /// backedge of the specified loop will execute if its exit condition
04647 /// were a conditional branch of the ICmpInst ExitCond, TBB, and FBB.
04648 ScalarEvolution::ExitLimit
04649 ScalarEvolution::ComputeExitLimitFromICmp(const Loop *L,
04650                                           ICmpInst *ExitCond,
04651                                           BasicBlock *TBB,
04652                                           BasicBlock *FBB,
04653                                           bool IsSubExpr) {
04654 
04655   // If the condition was exit on true, convert the condition to exit on false
04656   ICmpInst::Predicate Cond;
04657   if (!L->contains(FBB))
04658     Cond = ExitCond->getPredicate();
04659   else
04660     Cond = ExitCond->getInversePredicate();
04661 
04662   // Handle common loops like: for (X = "string"; *X; ++X)
04663   if (LoadInst *LI = dyn_cast<LoadInst>(ExitCond->getOperand(0)))
04664     if (Constant *RHS = dyn_cast<Constant>(ExitCond->getOperand(1))) {
04665       ExitLimit ItCnt =
04666         ComputeLoadConstantCompareExitLimit(LI, RHS, L, Cond);
04667       if (ItCnt.hasAnyInfo())
04668         return ItCnt;
04669     }
04670 
04671   const SCEV *LHS = getSCEV(ExitCond->getOperand(0));
04672   const SCEV *RHS = getSCEV(ExitCond->getOperand(1));
04673 
04674   // Try to evaluate any dependencies out of the loop.
04675   LHS = getSCEVAtScope(LHS, L);
04676   RHS = getSCEVAtScope(RHS, L);
04677 
04678   // At this point, we would like to compute how many iterations of the
04679   // loop the predicate will return true for these inputs.
04680   if (isLoopInvariant(LHS, L) && !isLoopInvariant(RHS, L)) {
04681     // If there is a loop-invariant, force it into the RHS.
04682     std::swap(LHS, RHS);
04683     Cond = ICmpInst::getSwappedPredicate(Cond);
04684   }
04685 
04686   // Simplify the operands before analyzing them.
04687   (void)SimplifyICmpOperands(Cond, LHS, RHS);
04688 
04689   // If we have a comparison of a chrec against a constant, try to use value
04690   // ranges to answer this query.
04691   if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS))
04692     if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(LHS))
04693       if (AddRec->getLoop() == L) {
04694         // Form the constant range.
04695         ConstantRange CompRange(
04696             ICmpInst::makeConstantRange(Cond, RHSC->getValue()->getValue()));
04697 
04698         const SCEV *Ret = AddRec->getNumIterationsInRange(CompRange, *this);
04699         if (!isa<SCEVCouldNotCompute>(Ret)) return Ret;
04700       }
04701 
04702   switch (Cond) {
04703   case ICmpInst::ICMP_NE: {                     // while (X != Y)
04704     // Convert to: while (X-Y != 0)
04705     ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, IsSubExpr);
04706     if (EL.hasAnyInfo()) return EL;
04707     break;
04708   }
04709   case ICmpInst::ICMP_EQ: {                     // while (X == Y)
04710     // Convert to: while (X-Y == 0)
04711     ExitLimit EL = HowFarToNonZero(getMinusSCEV(LHS, RHS), L);
04712     if (EL.hasAnyInfo()) return EL;
04713     break;
04714   }
04715   case ICmpInst::ICMP_SLT:
04716   case ICmpInst::ICMP_ULT: {                    // while (X < Y)
04717     bool IsSigned = Cond == ICmpInst::ICMP_SLT;
04718     ExitLimit EL = HowManyLessThans(LHS, RHS, L, IsSigned, IsSubExpr);
04719     if (EL.hasAnyInfo()) return EL;
04720     break;
04721   }
04722   case ICmpInst::ICMP_SGT:
04723   case ICmpInst::ICMP_UGT: {                    // while (X > Y)
04724     bool IsSigned = Cond == ICmpInst::ICMP_SGT;
04725     ExitLimit EL = HowManyGreaterThans(LHS, RHS, L, IsSigned, IsSubExpr);
04726     if (EL.hasAnyInfo()) return EL;
04727     break;
04728   }
04729   default:
04730 #if 0
04731     dbgs() << "ComputeBackedgeTakenCount ";
04732     if (ExitCond->getOperand(0)->getType()->isUnsigned())
04733       dbgs() << "[unsigned] ";
04734     dbgs() << *LHS << "   "
04735          << Instruction::getOpcodeName(Instruction::ICmp)
04736          << "   " << *RHS << "\n";
04737 #endif
04738     break;
04739   }
04740   return ComputeExitCountExhaustively(L, ExitCond, !L->contains(TBB));
04741 }
04742 
04743 ScalarEvolution::ExitLimit
04744 ScalarEvolution::ComputeExitLimitFromSingleExitSwitch(const Loop *L,
04745                                                       SwitchInst *Switch,
04746                                                       BasicBlock *ExitingBlock,
04747                                                       bool IsSubExpr) {
04748   assert(!L->contains(ExitingBlock) && "Not an exiting block!");
04749 
04750   // Give up if the exit is the default dest of a switch.
04751   if (Switch->getDefaultDest() == ExitingBlock)
04752     return getCouldNotCompute();
04753 
04754   assert(L->contains(Switch->getDefaultDest()) &&
04755          "Default case must not exit the loop!");
04756   const SCEV *LHS = getSCEVAtScope(Switch->getCondition(), L);
04757   const SCEV *RHS = getConstant(Switch->findCaseDest(ExitingBlock));
04758 
04759   // while (X != Y) --> while (X-Y != 0)
04760   ExitLimit EL = HowFarToZero(getMinusSCEV(LHS, RHS), L, IsSubExpr);
04761   if (EL.hasAnyInfo())
04762     return EL;
04763 
04764   return getCouldNotCompute();
04765 }
04766 
04767 static ConstantInt *
04768 EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
04769                                 ScalarEvolution &SE) {
04770   const SCEV *InVal = SE.getConstant(C);
04771   const SCEV *Val = AddRec->evaluateAtIteration(InVal, SE);
04772   assert(isa<SCEVConstant>(Val) &&
04773          "Evaluation of SCEV at constant didn't fold correctly?");
04774   return cast<SCEVConstant>(Val)->getValue();
04775 }
04776 
04777 /// ComputeLoadConstantCompareExitLimit - Given an exit condition of
04778 /// 'icmp op load X, cst', try to see if we can compute the backedge
04779 /// execution count.
04780 ScalarEvolution::ExitLimit
04781 ScalarEvolution::ComputeLoadConstantCompareExitLimit(
04782   LoadInst *LI,
04783   Constant *RHS,
04784   const Loop *L,
04785   ICmpInst::Predicate predicate) {
04786 
04787   if (LI->isVolatile()) return getCouldNotCompute();
04788 
04789   // Check to see if the loaded pointer is a getelementptr of a global.
04790   // TODO: Use SCEV instead of manually grubbing with GEPs.
04791   GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0));
04792   if (!GEP) return getCouldNotCompute();
04793 
04794   // Make sure that it is really a constant global we are gepping, with an
04795   // initializer, and make sure the first IDX is really 0.
04796   GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0));
04797   if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() ||
04798       GEP->getNumOperands() < 3 || !isa<Constant>(GEP->getOperand(1)) ||
04799       !cast<Constant>(GEP->getOperand(1))->isNullValue())
04800     return getCouldNotCompute();
04801 
04802   // Okay, we allow one non-constant index into the GEP instruction.
04803   Value *VarIdx = nullptr;
04804   std::vector<Constant*> Indexes;
04805   unsigned VarIdxNum = 0;
04806   for (unsigned i = 2, e = GEP->getNumOperands(); i != e; ++i)
04807     if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP->getOperand(i))) {
04808       Indexes.push_back(CI);
04809     } else if (!isa<ConstantInt>(GEP->getOperand(i))) {
04810       if (VarIdx) return getCouldNotCompute();  // Multiple non-constant idx's.
04811       VarIdx = GEP->getOperand(i);
04812       VarIdxNum = i-2;
04813       Indexes.push_back(nullptr);
04814     }
04815 
04816   // Loop-invariant loads may be a byproduct of loop optimization. Skip them.
04817   if (!VarIdx)
04818     return getCouldNotCompute();
04819 
04820   // Okay, we know we have a (load (gep GV, 0, X)) comparison with a constant.
04821   // Check to see if X is a loop variant variable value now.
04822   const SCEV *Idx = getSCEV(VarIdx);
04823   Idx = getSCEVAtScope(Idx, L);
04824 
04825   // We can only recognize very limited forms of loop index expressions, in
04826   // particular, only affine AddRec's like {C1,+,C2}.
04827   const SCEVAddRecExpr *IdxExpr = dyn_cast<SCEVAddRecExpr>(Idx);
04828   if (!IdxExpr || !IdxExpr->isAffine() || isLoopInvariant(IdxExpr, L) ||
04829       !isa<SCEVConstant>(IdxExpr->getOperand(0)) ||
04830       !isa<SCEVConstant>(IdxExpr->getOperand(1)))
04831     return getCouldNotCompute();
04832 
04833   unsigned MaxSteps = MaxBruteForceIterations;
04834   for (unsigned IterationNum = 0; IterationNum != MaxSteps; ++IterationNum) {
04835     ConstantInt *ItCst = ConstantInt::get(
04836                            cast<IntegerType>(IdxExpr->getType()), IterationNum);
04837     ConstantInt *Val = EvaluateConstantChrecAtConstant(IdxExpr, ItCst, *this);
04838 
04839     // Form the GEP offset.
04840     Indexes[VarIdxNum] = Val;
04841 
04842     Constant *Result = ConstantFoldLoadThroughGEPIndices(GV->getInitializer(),
04843                                                          Indexes);
04844     if (!Result) break;  // Cannot compute!
04845 
04846     // Evaluate the condition for this iteration.
04847     Result = ConstantExpr::getICmp(predicate, Result, RHS);
04848     if (!isa<ConstantInt>(Result)) break;  // Couldn't decide for sure
04849     if (cast<ConstantInt>(Result)->getValue().isMinValue()) {
04850 #if 0
04851       dbgs() << "\n***\n*** Computed loop count " << *ItCst
04852              << "\n*** From global " << *GV << "*** BB: " << *L->getHeader()
04853              << "***\n";
04854 #endif
04855       ++NumArrayLenItCounts;
04856       return getConstant(ItCst);   // Found terminating iteration!
04857     }
04858   }
04859   return getCouldNotCompute();
04860 }
04861 
04862 
04863 /// CanConstantFold - Return true if we can constant fold an instruction of the
04864 /// specified type, assuming that all operands were constants.
04865 static bool CanConstantFold(const Instruction *I) {
04866   if (isa<BinaryOperator>(I) || isa<CmpInst>(I) ||
04867       isa<SelectInst>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
04868       isa<LoadInst>(I))
04869     return true;
04870 
04871   if (const CallInst *CI = dyn_cast<CallInst>(I))
04872     if (const Function *F = CI->getCalledFunction())
04873       return canConstantFoldCallTo(F);
04874   return false;
04875 }
04876 
04877 /// Determine whether this instruction can constant evolve within this loop
04878 /// assuming its operands can all constant evolve.
04879 static bool canConstantEvolve(Instruction *I, const Loop *L) {
04880   // An instruction outside of the loop can't be derived from a loop PHI.
04881   if (!L->contains(I)) return false;
04882 
04883   if (isa<PHINode>(I)) {
04884     if (L->getHeader() == I->getParent())
04885       return true;
04886     else
04887       // We don't currently keep track of the control flow needed to evaluate
04888       // PHIs, so we cannot handle PHIs inside of loops.
04889       return false;
04890   }
04891 
04892   // If we won't be able to constant fold this expression even if the operands
04893   // are constants, bail early.
04894   return CanConstantFold(I);
04895 }
04896 
04897 /// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
04898 /// recursing through each instruction operand until reaching a loop header phi.
04899 static PHINode *
04900 getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L,
04901                                DenseMap<Instruction *, PHINode *> &PHIMap) {
04902 
04903   // Otherwise, we can evaluate this instruction if all of its operands are
04904   // constant or derived from a PHI node themselves.
04905   PHINode *PHI = nullptr;
04906   for (Instruction::op_iterator OpI = UseInst->op_begin(),
04907          OpE = UseInst->op_end(); OpI != OpE; ++OpI) {
04908 
04909     if (isa<Constant>(*OpI)) continue;
04910 
04911     Instruction *OpInst = dyn_cast<Instruction>(*OpI);
04912     if (!OpInst || !canConstantEvolve(OpInst, L)) return nullptr;
04913 
04914     PHINode *P = dyn_cast<PHINode>(OpInst);
04915     if (!P)
04916       // If this operand is already visited, reuse the prior result.
04917       // We may have P != PHI if this is the deepest point at which the
04918       // inconsistent paths meet.
04919       P = PHIMap.lookup(OpInst);
04920     if (!P) {
04921       // Recurse and memoize the results, whether a phi is found or not.
04922       // This recursive call invalidates pointers into PHIMap.
04923       P = getConstantEvolvingPHIOperands(OpInst, L, PHIMap);
04924       PHIMap[OpInst] = P;
04925     }
04926     if (!P)
04927       return nullptr;  // Not evolving from PHI
04928     if (PHI && PHI != P)
04929       return nullptr;  // Evolving from multiple different PHIs.
04930     PHI = P;
04931   }
04932   // This is a expression evolving from a constant PHI!
04933   return PHI;
04934 }
04935 
04936 /// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
04937 /// in the loop that V is derived from.  We allow arbitrary operations along the
04938 /// way, but the operands of an operation must either be constants or a value
04939 /// derived from a constant PHI.  If this expression does not fit with these
04940 /// constraints, return null.
04941 static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) {
04942   Instruction *I = dyn_cast<Instruction>(V);
04943   if (!I || !canConstantEvolve(I, L)) return nullptr;
04944 
04945   if (PHINode *PN = dyn_cast<PHINode>(I)) {
04946     return PN;
04947   }
04948 
04949   // Record non-constant instructions contained by the loop.
04950   DenseMap<Instruction *, PHINode *> PHIMap;
04951   return getConstantEvolvingPHIOperands(I, L, PHIMap);
04952 }
04953 
04954 /// EvaluateExpression - Given an expression that passes the
04955 /// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
04956 /// in the loop has the value PHIVal.  If we can't fold this expression for some
04957 /// reason, return null.
04958 static Constant *EvaluateExpression(Value *V, const Loop *L,
04959                                     DenseMap<Instruction *, Constant *> &Vals,
04960                                     const DataLayout *DL,
04961                                     const TargetLibraryInfo *TLI) {
04962   // Convenient constant check, but redundant for recursive calls.
04963   if (Constant *C = dyn_cast<Constant>(V)) return C;
04964   Instruction *I = dyn_cast<Instruction>(V);
04965   if (!I) return nullptr;
04966 
04967   if (Constant *C = Vals.lookup(I)) return C;
04968 
04969   // An instruction inside the loop depends on a value outside the loop that we
04970   // weren't given a mapping for, or a value such as a call inside the loop.
04971   if (!canConstantEvolve(I, L)) return nullptr;
04972 
04973   // An unmapped PHI can be due to a branch or another loop inside this loop,
04974   // or due to this not being the initial iteration through a loop where we
04975   // couldn't compute the evolution of this particular PHI last time.
04976   if (isa<PHINode>(I)) return nullptr;
04977 
04978   std::vector<Constant*> Operands(I->getNumOperands());
04979 
04980   for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
04981     Instruction *Operand = dyn_cast<Instruction>(I->getOperand(i));
04982     if (!Operand) {
04983       Operands[i] = dyn_cast<Constant>(I->getOperand(i));
04984       if (!Operands[i]) return nullptr;
04985       continue;
04986     }
04987     Constant *C = EvaluateExpression(Operand, L, Vals, DL, TLI);
04988     Vals[Operand] = C;
04989     if (!C) return nullptr;
04990     Operands[i] = C;
04991   }
04992 
04993   if (CmpInst *CI = dyn_cast<CmpInst>(I))
04994     return ConstantFoldCompareInstOperands(CI->getPredicate(), Operands[0],
04995                                            Operands[1], DL, TLI);
04996   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
04997     if (!LI->isVolatile())
04998       return ConstantFoldLoadFromConstPtr(Operands[0], DL);
04999   }
05000   return ConstantFoldInstOperands(I->getOpcode(), I->getType(), Operands, DL,
05001                                   TLI);
05002 }
05003 
05004 /// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
05005 /// in the header of its containing loop, we know the loop executes a
05006 /// constant number of times, and the PHI node is just a recurrence
05007 /// involving constants, fold it.
05008 Constant *
05009 ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
05010                                                    const APInt &BEs,
05011                                                    const Loop *L) {
05012   DenseMap<PHINode*, Constant*>::const_iterator I =
05013     ConstantEvolutionLoopExitValue.find(PN);
05014   if (I != ConstantEvolutionLoopExitValue.end())
05015     return I->second;
05016 
05017   if (BEs.ugt(MaxBruteForceIterations))
05018     return ConstantEvolutionLoopExitValue[PN] = nullptr;  // Not going to evaluate it.
05019 
05020   Constant *&RetVal = ConstantEvolutionLoopExitValue[PN];
05021 
05022   DenseMap<Instruction *, Constant *> CurrentIterVals;
05023   BasicBlock *Header = L->getHeader();
05024   assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
05025 
05026   // Since the loop is canonicalized, the PHI node must have two entries.  One
05027   // entry must be a constant (coming in from outside of the loop), and the
05028   // second must be derived from the same PHI.
05029   bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1));
05030   PHINode *PHI = nullptr;
05031   for (BasicBlock::iterator I = Header->begin();
05032        (PHI = dyn_cast<PHINode>(I)); ++I) {
05033     Constant *StartCST =
05034       dyn_cast<Constant>(PHI->getIncomingValue(!SecondIsBackedge));
05035     if (!StartCST) continue;
05036     CurrentIterVals[PHI] = StartCST;
05037   }
05038   if (!CurrentIterVals.count(PN))
05039     return RetVal = nullptr;
05040 
05041   Value *BEValue = PN->getIncomingValue(SecondIsBackedge);
05042 
05043   // Execute the loop symbolically to determine the exit value.
05044   if (BEs.getActiveBits() >= 32)
05045     return RetVal = nullptr; // More than 2^32-1 iterations?? Not doing it!
05046 
05047   unsigned NumIterations = BEs.getZExtValue(); // must be in range
05048   unsigned IterationNum = 0;
05049   for (; ; ++IterationNum) {
05050     if (IterationNum == NumIterations)
05051       return RetVal = CurrentIterVals[PN];  // Got exit value!
05052 
05053     // Compute the value of the PHIs for the next iteration.
05054     // EvaluateExpression adds non-phi values to the CurrentIterVals map.
05055     DenseMap<Instruction *, Constant *> NextIterVals;
05056     Constant *NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL,
05057                                            TLI);
05058     if (!NextPHI)
05059       return nullptr;        // Couldn't evaluate!
05060     NextIterVals[PN] = NextPHI;
05061 
05062     bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
05063 
05064     // Also evaluate the other PHI nodes.  However, we don't get to stop if we
05065     // cease to be able to evaluate one of them or if they stop evolving,
05066     // because that doesn't necessarily prevent us from computing PN.
05067     SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute;
05068     for (DenseMap<Instruction *, Constant *>::const_iterator
05069            I = CurrentIterVals.begin(), E = CurrentIterVals.end(); I != E; ++I){
05070       PHINode *PHI = dyn_cast<PHINode>(I->first);
05071       if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
05072       PHIsToCompute.push_back(std::make_pair(PHI, I->second));
05073     }
05074     // We use two distinct loops because EvaluateExpression may invalidate any
05075     // iterators into CurrentIterVals.
05076     for (SmallVectorImpl<std::pair<PHINode *, Constant*> >::const_iterator
05077              I = PHIsToCompute.begin(), E = PHIsToCompute.end(); I != E; ++I) {
05078       PHINode *PHI = I->first;
05079       Constant *&NextPHI = NextIterVals[PHI];
05080       if (!NextPHI) {   // Not already computed.
05081         Value *BEValue = PHI->getIncomingValue(SecondIsBackedge);
05082         NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, TLI);
05083       }
05084       if (NextPHI != I->second)
05085         StoppedEvolving = false;
05086     }
05087 
05088     // If all entries in CurrentIterVals == NextIterVals then we can stop
05089     // iterating, the loop can't continue to change.
05090     if (StoppedEvolving)
05091       return RetVal = CurrentIterVals[PN];
05092 
05093     CurrentIterVals.swap(NextIterVals);
05094   }
05095 }
05096 
05097 /// ComputeExitCountExhaustively - If the loop is known to execute a
05098 /// constant number of times (the condition evolves only from constants),
05099 /// try to evaluate a few iterations of the loop until we get the exit
05100 /// condition gets a value of ExitWhen (true or false).  If we cannot
05101 /// evaluate the trip count of the loop, return getCouldNotCompute().
05102 const SCEV *ScalarEvolution::ComputeExitCountExhaustively(const Loop *L,
05103                                                           Value *Cond,
05104                                                           bool ExitWhen) {
05105   PHINode *PN = getConstantEvolvingPHI(Cond, L);
05106   if (!PN) return getCouldNotCompute();
05107 
05108   // If the loop is canonicalized, the PHI will have exactly two entries.
05109   // That's the only form we support here.
05110   if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
05111 
05112   DenseMap<Instruction *, Constant *> CurrentIterVals;
05113   BasicBlock *Header = L->getHeader();
05114   assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
05115 
05116   // One entry must be a constant (coming in from outside of the loop), and the
05117   // second must be derived from the same PHI.
05118   bool SecondIsBackedge = L->contains(PN->getIncomingBlock(1));
05119   PHINode *PHI = nullptr;
05120   for (BasicBlock::iterator I = Header->begin();
05121        (PHI = dyn_cast<PHINode>(I)); ++I) {
05122     Constant *StartCST =
05123       dyn_cast<Constant>(PHI->getIncomingValue(!SecondIsBackedge));
05124     if (!StartCST) continue;
05125     CurrentIterVals[PHI] = StartCST;
05126   }
05127   if (!CurrentIterVals.count(PN))
05128     return getCouldNotCompute();
05129 
05130   // Okay, we find a PHI node that defines the trip count of this loop.  Execute
05131   // the loop symbolically to determine when the condition gets a value of
05132   // "ExitWhen".
05133 
05134   unsigned MaxIterations = MaxBruteForceIterations;   // Limit analysis.
05135   for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
05136     ConstantInt *CondVal =
05137       dyn_cast_or_null<ConstantInt>(EvaluateExpression(Cond, L, CurrentIterVals,
05138                                                        DL, TLI));
05139 
05140     // Couldn't symbolically evaluate.
05141     if (!CondVal) return getCouldNotCompute();
05142 
05143     if (CondVal->getValue() == uint64_t(ExitWhen)) {
05144       ++NumBruteForceTripCountsComputed;
05145       return getConstant(Type::getInt32Ty(getContext()), IterationNum);
05146     }
05147 
05148     // Update all the PHI nodes for the next iteration.
05149     DenseMap<Instruction *, Constant *> NextIterVals;
05150 
05151     // Create a list of which PHIs we need to compute. We want to do this before
05152     // calling EvaluateExpression on them because that may invalidate iterators
05153     // into CurrentIterVals.
05154     SmallVector<PHINode *, 8> PHIsToCompute;
05155     for (DenseMap<Instruction *, Constant *>::const_iterator
05156            I = CurrentIterVals.begin(), E = CurrentIterVals.end(); I != E; ++I){
05157       PHINode *PHI = dyn_cast<PHINode>(I->first);
05158       if (!PHI || PHI->getParent() != Header) continue;
05159       PHIsToCompute.push_back(PHI);
05160     }
05161     for (SmallVectorImpl<PHINode *>::const_iterator I = PHIsToCompute.begin(),
05162              E = PHIsToCompute.end(); I != E; ++I) {
05163       PHINode *PHI = *I;
05164       Constant *&NextPHI = NextIterVals[PHI];
05165       if (NextPHI) continue;    // Already computed!
05166 
05167       Value *BEValue = PHI->getIncomingValue(SecondIsBackedge);
05168       NextPHI = EvaluateExpression(BEValue, L, CurrentIterVals, DL, TLI);
05169     }
05170     CurrentIterVals.swap(NextIterVals);
05171   }
05172 
05173   // Too many iterations were needed to evaluate.
05174   return getCouldNotCompute();
05175 }
05176 
05177 /// getSCEVAtScope - Return a SCEV expression for the specified value
05178 /// at the specified scope in the program.  The L value specifies a loop
05179 /// nest to evaluate the expression at, where null is the top-level or a
05180 /// specified loop is immediately inside of the loop.
05181 ///
05182 /// This method can be used to compute the exit value for a variable defined
05183 /// in a loop by querying what the value will hold in the parent loop.
05184 ///
05185 /// In the case that a relevant loop exit value cannot be computed, the
05186 /// original value V is returned.
05187 const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
05188   // Check to see if we've folded this expression at this loop before.
05189   SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values = ValuesAtScopes[V];
05190   for (unsigned u = 0; u < Values.size(); u++) {
05191     if (Values[u].first == L)
05192       return Values[u].second ? Values[u].second : V;
05193   }
05194   Values.push_back(std::make_pair(L, static_cast<const SCEV *>(nullptr)));
05195   // Otherwise compute it.
05196   const SCEV *C = computeSCEVAtScope(V, L);
05197   SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values2 = ValuesAtScopes[V];
05198   for (unsigned u = Values2.size(); u > 0; u--) {
05199     if (Values2[u - 1].first == L) {
05200       Values2[u - 1].second = C;
05201       break;
05202     }
05203   }
05204   return C;
05205 }
05206 
05207 /// This builds up a Constant using the ConstantExpr interface.  That way, we
05208 /// will return Constants for objects which aren't represented by a
05209 /// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
05210 /// Returns NULL if the SCEV isn't representable as a Constant.
05211 static Constant *BuildConstantFromSCEV(const SCEV *V) {
05212   switch (static_cast<SCEVTypes>(V->getSCEVType())) {
05213     case scCouldNotCompute:
05214     case scAddRecExpr:
05215       break;
05216     case scConstant:
05217       return cast<SCEVConstant>(V)->getValue();
05218     case scUnknown:
05219       return dyn_cast<Constant>(cast<SCEVUnknown>(V)->getValue());
05220     case scSignExtend: {
05221       const SCEVSignExtendExpr *SS = cast<SCEVSignExtendExpr>(V);
05222       if (Constant *CastOp = BuildConstantFromSCEV(SS->getOperand()))
05223         return ConstantExpr::getSExt(CastOp, SS->getType());
05224       break;
05225     }
05226     case scZeroExtend: {
05227       const SCEVZeroExtendExpr *SZ = cast<SCEVZeroExtendExpr>(V);
05228       if (Constant *CastOp = BuildConstantFromSCEV(SZ->getOperand()))
05229         return ConstantExpr::getZExt(CastOp, SZ->getType());
05230       break;
05231     }
05232     case scTruncate: {
05233       const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(V);
05234       if (Constant *CastOp = BuildConstantFromSCEV(ST->getOperand()))
05235         return ConstantExpr::getTrunc(CastOp, ST->getType());
05236       break;
05237     }
05238     case scAddExpr: {
05239       const SCEVAddExpr *SA = cast<SCEVAddExpr>(V);
05240       if (Constant *C = BuildConstantFromSCEV(SA->getOperand(0))) {
05241         if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) {
05242           unsigned AS = PTy->getAddressSpace();
05243           Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS);
05244           C = ConstantExpr::getBitCast(C, DestPtrTy);
05245         }
05246         for (unsigned i = 1, e = SA->getNumOperands(); i != e; ++i) {
05247           Constant *C2 = BuildConstantFromSCEV(SA->getOperand(i));
05248           if (!C2) return nullptr;
05249 
05250           // First pointer!
05251           if (!C->getType()->isPointerTy() && C2->getType()->isPointerTy()) {
05252             unsigned AS = C2->getType()->getPointerAddressSpace();
05253             std::swap(C, C2);
05254             Type *DestPtrTy = Type::getInt8PtrTy(C->getContext(), AS);
05255             // The offsets have been converted to bytes.  We can add bytes to an
05256             // i8* by GEP with the byte count in the first index.
05257             C = ConstantExpr::getBitCast(C, DestPtrTy);
05258           }
05259 
05260           // Don't bother trying to sum two pointers. We probably can't
05261           // statically compute a load that results from it anyway.
05262           if (C2->getType()->isPointerTy())
05263             return nullptr;
05264 
05265           if (PointerType *PTy = dyn_cast<PointerType>(C->getType())) {
05266             if (PTy->getElementType()->isStructTy())
05267               C2 = ConstantExpr::getIntegerCast(
05268                   C2, Type::getInt32Ty(C->getContext()), true);
05269             C = ConstantExpr::getGetElementPtr(C, C2);
05270           } else
05271             C = ConstantExpr::getAdd(C, C2);
05272         }
05273         return C;
05274       }
05275       break;
05276     }
05277     case scMulExpr: {
05278       const SCEVMulExpr *SM = cast<SCEVMulExpr>(V);
05279       if (Constant *C = BuildConstantFromSCEV(SM->getOperand(0))) {
05280         // Don't bother with pointers at all.
05281         if (C->getType()->isPointerTy()) return nullptr;
05282         for (unsigned i = 1, e = SM->getNumOperands(); i != e; ++i) {
05283           Constant *C2 = BuildConstantFromSCEV(SM->getOperand(i));
05284           if (!C2 || C2->getType()->isPointerTy()) return nullptr;
05285           C = ConstantExpr::getMul(C, C2);
05286         }
05287         return C;
05288       }
05289       break;
05290     }
05291     case scUDivExpr: {
05292       const SCEVUDivExpr *SU = cast<SCEVUDivExpr>(V);
05293       if (Constant *LHS = BuildConstantFromSCEV(SU->getLHS()))
05294         if (Constant *RHS = BuildConstantFromSCEV(SU->getRHS()))
05295           if (LHS->getType() == RHS->getType())
05296             return ConstantExpr::getUDiv(LHS, RHS);
05297       break;
05298     }
05299     case scSMaxExpr:
05300     case scUMaxExpr:
05301       break; // TODO: smax, umax.
05302   }
05303   return nullptr;
05304 }
05305 
05306 const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
05307   if (isa<SCEVConstant>(V)) return V;
05308 
05309   // If this instruction is evolved from a constant-evolving PHI, compute the
05310   // exit value from the loop without using SCEVs.
05311   if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) {
05312     if (Instruction *I = dyn_cast<Instruction>(SU->getValue())) {
05313       const Loop *LI = (*this->LI)[I->getParent()];
05314       if (LI && LI->getParentLoop() == L)  // Looking for loop exit value.
05315         if (PHINode *PN = dyn_cast<PHINode>(I))
05316           if (PN->getParent() == LI->getHeader()) {
05317             // Okay, there is no closed form solution for the PHI node.  Check
05318             // to see if the loop that contains it has a known backedge-taken
05319             // count.  If so, we may be able to force computation of the exit
05320             // value.
05321             const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI);
05322             if (const SCEVConstant *BTCC =
05323                   dyn_cast<SCEVConstant>(BackedgeTakenCount)) {
05324               // Okay, we know how many times the containing loop executes.  If
05325               // this is a constant evolving PHI node, get the final value at
05326               // the specified iteration number.
05327               Constant *RV = getConstantEvolutionLoopExitValue(PN,
05328                                                    BTCC->getValue()->getValue(),
05329                                                                LI);
05330               if (RV) return getSCEV(RV);
05331             }
05332           }
05333 
05334       // Okay, this is an expression that we cannot symbolically evaluate
05335       // into a SCEV.  Check to see if it's possible to symbolically evaluate
05336       // the arguments into constants, and if so, try to constant propagate the
05337       // result.  This is particularly useful for computing loop exit values.
05338       if (CanConstantFold(I)) {
05339         SmallVector<Constant *, 4> Operands;
05340         bool MadeImprovement = false;
05341         for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
05342           Value *Op = I->getOperand(i);
05343           if (Constant *C = dyn_cast<Constant>(Op)) {
05344             Operands.push_back(C);
05345             continue;
05346           }
05347 
05348           // If any of the operands is non-constant and if they are
05349           // non-integer and non-pointer, don't even try to analyze them
05350           // with scev techniques.
05351           if (!isSCEVable(Op->getType()))
05352             return V;
05353 
05354           const SCEV *OrigV = getSCEV(Op);
05355           const SCEV *OpV = getSCEVAtScope(OrigV, L);
05356           MadeImprovement |= OrigV != OpV;
05357 
05358           Constant *C = BuildConstantFromSCEV(OpV);
05359           if (!C) return V;
05360           if (C->getType() != Op->getType())
05361             C = ConstantExpr::getCast(CastInst::getCastOpcode(C, false,
05362                                                               Op->getType(),
05363                                                               false),
05364                                       C, Op->getType());
05365           Operands.push_back(C);
05366         }
05367 
05368         // Check to see if getSCEVAtScope actually made an improvement.
05369         if (MadeImprovement) {
05370           Constant *C = nullptr;
05371           if (const CmpInst *CI = dyn_cast<CmpInst>(I))
05372             C = ConstantFoldCompareInstOperands(CI->getPredicate(),
05373                                                 Operands[0], Operands[1], DL,
05374                                                 TLI);
05375           else if (const LoadInst *LI = dyn_cast<LoadInst>(I)) {
05376             if (!LI->isVolatile())
05377               C = ConstantFoldLoadFromConstPtr(Operands[0], DL);
05378           } else
05379             C = ConstantFoldInstOperands(I->getOpcode(), I->getType(),
05380                                          Operands, DL, TLI);
05381           if (!C) return V;
05382           return getSCEV(C);
05383         }
05384       }
05385     }
05386 
05387     // This is some other type of SCEVUnknown, just return it.
05388     return V;
05389   }
05390 
05391   if (const SCEVCommutativeExpr *Comm = dyn_cast<SCEVCommutativeExpr>(V)) {
05392     // Avoid performing the look-up in the common case where the specified
05393     // expression has no loop-variant portions.
05394     for (unsigned i = 0, e = Comm->getNumOperands(); i != e; ++i) {
05395       const SCEV *OpAtScope = getSCEVAtScope(Comm->getOperand(i), L);
05396       if (OpAtScope != Comm->getOperand(i)) {
05397         // Okay, at least one of these operands is loop variant but might be
05398         // foldable.  Build a new instance of the folded commutative expression.
05399         SmallVector<const SCEV *, 8> NewOps(Comm->op_begin(),
05400                                             Comm->op_begin()+i);
05401         NewOps.push_back(OpAtScope);
05402 
05403         for (++i; i != e; ++i) {
05404           OpAtScope = getSCEVAtScope(Comm->getOperand(i), L);
05405           NewOps.push_back(OpAtScope);
05406         }
05407         if (isa<SCEVAddExpr>(Comm))
05408           return getAddExpr(NewOps);
05409         if (isa<SCEVMulExpr>(Comm))
05410           return getMulExpr(NewOps);
05411         if (isa<SCEVSMaxExpr>(Comm))
05412           return getSMaxExpr(NewOps);
05413         if (isa<SCEVUMaxExpr>(Comm))
05414           return getUMaxExpr(NewOps);
05415         llvm_unreachable("Unknown commutative SCEV type!");
05416       }
05417     }
05418     // If we got here, all operands are loop invariant.
05419     return Comm;
05420   }
05421 
05422   if (const SCEVUDivExpr *Div = dyn_cast<SCEVUDivExpr>(V)) {
05423     const SCEV *LHS = getSCEVAtScope(Div->getLHS(), L);
05424     const SCEV *RHS = getSCEVAtScope(Div->getRHS(), L);
05425     if (LHS == Div->getLHS() && RHS == Div->getRHS())
05426       return Div;   // must be loop invariant
05427     return getUDivExpr(LHS, RHS);
05428   }
05429 
05430   // If this is a loop recurrence for a loop that does not contain L, then we
05431   // are dealing with the final value computed by the loop.
05432   if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V)) {
05433     // First, attempt to evaluate each operand.
05434     // Avoid performing the look-up in the common case where the specified
05435     // expression has no loop-variant portions.
05436     for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
05437       const SCEV *OpAtScope = getSCEVAtScope(AddRec->getOperand(i), L);
05438       if (OpAtScope == AddRec->getOperand(i))
05439         continue;
05440 
05441       // Okay, at least one of these operands is loop variant but might be
05442       // foldable.  Build a new instance of the folded commutative expression.
05443       SmallVector<const SCEV *, 8> NewOps(AddRec->op_begin(),
05444                                           AddRec->op_begin()+i);
05445       NewOps.push_back(OpAtScope);
05446       for (++i; i != e; ++i)
05447         NewOps.push_back(getSCEVAtScope(AddRec->getOperand(i), L));
05448 
05449       const SCEV *FoldedRec =
05450         getAddRecExpr(NewOps, AddRec->getLoop(),
05451                       AddRec->getNoWrapFlags(SCEV::FlagNW));
05452       AddRec = dyn_cast<SCEVAddRecExpr>(FoldedRec);
05453       // The addrec may be folded to a nonrecurrence, for example, if the
05454       // induction variable is multiplied by zero after constant folding. Go
05455       // ahead and return the folded value.
05456       if (!AddRec)
05457         return FoldedRec;
05458       break;
05459     }
05460 
05461     // If the scope is outside the addrec's loop, evaluate it by using the
05462     // loop exit value of the addrec.
05463     if (!AddRec->getLoop()->contains(L)) {
05464       // To evaluate this recurrence, we need to know how many times the AddRec
05465       // loop iterates.  Compute this now.
05466       const SCEV *BackedgeTakenCount = getBackedgeTakenCount(AddRec->getLoop());
05467       if (BackedgeTakenCount == getCouldNotCompute()) return AddRec;
05468 
05469       // Then, evaluate the AddRec.
05470       return AddRec->evaluateAtIteration(BackedgeTakenCount, *this);
05471     }
05472 
05473     return AddRec;
05474   }
05475 
05476   if (const SCEVZeroExtendExpr *Cast = dyn_cast<SCEVZeroExtendExpr>(V)) {
05477     const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
05478     if (Op == Cast->getOperand())
05479       return Cast;  // must be loop invariant
05480     return getZeroExtendExpr(Op, Cast->getType());
05481   }
05482 
05483   if (const SCEVSignExtendExpr *Cast = dyn_cast<SCEVSignExtendExpr>(V)) {
05484     const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
05485     if (Op == Cast->getOperand())
05486       return Cast;  // must be loop invariant
05487     return getSignExtendExpr(Op, Cast->getType());
05488   }
05489 
05490   if (const SCEVTruncateExpr *Cast = dyn_cast<SCEVTruncateExpr>(V)) {
05491     const SCEV *Op = getSCEVAtScope(Cast->getOperand(), L);
05492     if (Op == Cast->getOperand())
05493       return Cast;  // must be loop invariant
05494     return getTruncateExpr(Op, Cast->getType());
05495   }
05496 
05497   llvm_unreachable("Unknown SCEV type!");
05498 }
05499 
05500 /// getSCEVAtScope - This is a convenience function which does
05501 /// getSCEVAtScope(getSCEV(V), L).
05502 const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) {
05503   return getSCEVAtScope(getSCEV(V), L);
05504 }
05505 
05506 /// SolveLinEquationWithOverflow - Finds the minimum unsigned root of the
05507 /// following equation:
05508 ///
05509 ///     A * X = B (mod N)
05510 ///
05511 /// where N = 2^BW and BW is the common bit width of A and B. The signedness of
05512 /// A and B isn't important.
05513 ///
05514 /// If the equation does not have a solution, SCEVCouldNotCompute is returned.
05515 static const SCEV *SolveLinEquationWithOverflow(const APInt &A, const APInt &B,
05516                                                ScalarEvolution &SE) {
05517   uint32_t BW = A.getBitWidth();
05518   assert(BW == B.getBitWidth() && "Bit widths must be the same.");
05519   assert(A != 0 && "A must be non-zero.");
05520 
05521   // 1. D = gcd(A, N)
05522   //
05523   // The gcd of A and N may have only one prime factor: 2. The number of
05524   // trailing zeros in A is its multiplicity
05525   uint32_t Mult2 = A.countTrailingZeros();
05526   // D = 2^Mult2
05527 
05528   // 2. Check if B is divisible by D.
05529   //
05530   // B is divisible by D if and only if the multiplicity of prime factor 2 for B
05531   // is not less than multiplicity of this prime factor for D.
05532   if (B.countTrailingZeros() < Mult2)
05533     return SE.getCouldNotCompute();
05534 
05535   // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
05536   // modulo (N / D).
05537   //
05538   // (N / D) may need BW+1 bits in its representation.  Hence, we'll use this
05539   // bit width during computations.
05540   APInt AD = A.lshr(Mult2).zext(BW + 1);  // AD = A / D
05541   APInt Mod(BW + 1, 0);
05542   Mod.setBit(BW - Mult2);  // Mod = N / D
05543   APInt I = AD.multiplicativeInverse(Mod);
05544 
05545   // 4. Compute the minimum unsigned root of the equation:
05546   // I * (B / D) mod (N / D)
05547   APInt Result = (I * B.lshr(Mult2).zext(BW + 1)).urem(Mod);
05548 
05549   // The result is guaranteed to be less than 2^BW so we may truncate it to BW
05550   // bits.
05551   return SE.getConstant(Result.trunc(BW));
05552 }
05553 
05554 /// SolveQuadraticEquation - Find the roots of the quadratic equation for the
05555 /// given quadratic chrec {L,+,M,+,N}.  This returns either the two roots (which
05556 /// might be the same) or two SCEVCouldNotCompute objects.
05557 ///
05558 static std::pair<const SCEV *,const SCEV *>
05559 SolveQuadraticEquation(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
05560   assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
05561   const SCEVConstant *LC = dyn_cast<SCEVConstant>(AddRec->getOperand(0));
05562   const SCEVConstant *MC = dyn_cast<SCEVConstant>(AddRec->getOperand(1));
05563   const SCEVConstant *NC = dyn_cast<SCEVConstant>(AddRec->getOperand(2));
05564 
05565   // We currently can only solve this if the coefficients are constants.
05566   if (!LC || !MC || !NC) {
05567     const SCEV *CNC = SE.getCouldNotCompute();
05568     return std::make_pair(CNC, CNC);
05569   }
05570 
05571   uint32_t BitWidth = LC->getValue()->getValue().getBitWidth();
05572   const APInt &L = LC->getValue()->getValue();
05573   const APInt &M = MC->getValue()->getValue();
05574   const APInt &N = NC->getValue()->getValue();
05575   APInt Two(BitWidth, 2);
05576   APInt Four(BitWidth, 4);
05577 
05578   {
05579     using namespace APIntOps;
05580     const APInt& C = L;
05581     // Convert from chrec coefficients to polynomial coefficients AX^2+BX+C
05582     // The B coefficient is M-N/2
05583     APInt B(M);
05584     B -= sdiv(N,Two);
05585 
05586     // The A coefficient is N/2
05587     APInt A(N.sdiv(Two));
05588 
05589     // Compute the B^2-4ac term.
05590     APInt SqrtTerm(B);
05591     SqrtTerm *= B;
05592     SqrtTerm -= Four * (A * C);
05593 
05594     if (SqrtTerm.isNegative()) {
05595       // The loop is provably infinite.
05596       const SCEV *CNC = SE.getCouldNotCompute();
05597       return std::make_pair(CNC, CNC);
05598     }
05599 
05600     // Compute sqrt(B^2-4ac). This is guaranteed to be the nearest
05601     // integer value or else APInt::sqrt() will assert.
05602     APInt SqrtVal(SqrtTerm.sqrt());
05603 
05604     // Compute the two solutions for the quadratic formula.
05605     // The divisions must be performed as signed divisions.
05606     APInt NegB(-B);
05607     APInt TwoA(A << 1);
05608     if (TwoA.isMinValue()) {
05609       const SCEV *CNC = SE.getCouldNotCompute();
05610       return std::make_pair(CNC, CNC);
05611     }
05612 
05613     LLVMContext &Context = SE.getContext();
05614 
05615     ConstantInt *Solution1 =
05616       ConstantInt::get(Context, (NegB + SqrtVal).sdiv(TwoA));
05617     ConstantInt *Solution2 =
05618       ConstantInt::get(Context, (NegB - SqrtVal).sdiv(TwoA));
05619 
05620     return std::make_pair(SE.getConstant(Solution1),
05621                           SE.getConstant(Solution2));
05622   } // end APIntOps namespace
05623 }
05624 
05625 /// HowFarToZero - Return the number of times a backedge comparing the specified
05626 /// value to zero will execute.  If not computable, return CouldNotCompute.
05627 ///
05628 /// This is only used for loops with a "x != y" exit test. The exit condition is
05629 /// now expressed as a single expression, V = x-y. So the exit test is
05630 /// effectively V != 0.  We know and take advantage of the fact that this
05631 /// expression only being used in a comparison by zero context.
05632 ScalarEvolution::ExitLimit
05633 ScalarEvolution::HowFarToZero(const SCEV *V, const Loop *L, bool IsSubExpr) {
05634   // If the value is a constant
05635   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
05636     // If the value is already zero, the branch will execute zero times.
05637     if (C->getValue()->isZero()) return C;
05638     return getCouldNotCompute();  // Otherwise it will loop infinitely.
05639   }
05640 
05641   const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(V);
05642   if (!AddRec || AddRec->getLoop() != L)
05643     return getCouldNotCompute();
05644 
05645   // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
05646   // the quadratic equation to solve it.
05647   if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
05648     std::pair<const SCEV *,const SCEV *> Roots =
05649       SolveQuadraticEquation(AddRec, *this);
05650     const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first);
05651     const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second);
05652     if (R1 && R2) {
05653 #if 0
05654       dbgs() << "HFTZ: " << *V << " - sol#1: " << *R1
05655              << "  sol#2: " << *R2 << "\n";
05656 #endif
05657       // Pick the smallest positive root value.
05658       if (ConstantInt *CB =
05659           dyn_cast<ConstantInt>(ConstantExpr::getICmp(CmpInst::ICMP_ULT,
05660                                                       R1->getValue(),
05661                                                       R2->getValue()))) {
05662         if (CB->getZExtValue() == false)
05663           std::swap(R1, R2);   // R1 is the minimum root now.
05664 
05665         // We can only use this value if the chrec ends up with an exact zero
05666         // value at this index.  When solving for "X*X != 5", for example, we
05667         // should not accept a root of 2.
05668         const SCEV *Val = AddRec->evaluateAtIteration(R1, *this);
05669         if (Val->isZero())
05670           return R1;  // We found a quadratic root!
05671       }
05672     }
05673     return getCouldNotCompute();
05674   }
05675 
05676   // Otherwise we can only handle this if it is affine.
05677   if (!AddRec->isAffine())
05678     return getCouldNotCompute();
05679 
05680   // If this is an affine expression, the execution count of this branch is
05681   // the minimum unsigned root of the following equation:
05682   //
05683   //     Start + Step*N = 0 (mod 2^BW)
05684   //
05685   // equivalent to:
05686   //
05687   //             Step*N = -Start (mod 2^BW)
05688   //
05689   // where BW is the common bit width of Start and Step.
05690 
05691   // Get the initial value for the loop.
05692   const SCEV *Start = getSCEVAtScope(AddRec->getStart(), L->getParentLoop());
05693   const SCEV *Step = getSCEVAtScope(AddRec->getOperand(1), L->getParentLoop());
05694 
05695   // For now we handle only constant steps.
05696   //
05697   // TODO: Handle a nonconstant Step given AddRec<NUW>. If the
05698   // AddRec is NUW, then (in an unsigned sense) it cannot be counting up to wrap
05699   // to 0, it must be counting down to equal 0. Consequently, N = Start / -Step.
05700   // We have not yet seen any such cases.
05701   const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Step);
05702   if (!StepC || StepC->getValue()->equalsInt(0))
05703     return getCouldNotCompute();
05704 
05705   // For positive steps (counting up until unsigned overflow):
05706   //   N = -Start/Step (as unsigned)
05707   // For negative steps (counting down to zero):
05708   //   N = Start/-Step
05709   // First compute the unsigned distance from zero in the direction of Step.
05710   bool CountDown = StepC->getValue()->getValue().isNegative();
05711   const SCEV *Distance = CountDown ? Start : getNegativeSCEV(Start);
05712 
05713   // Handle unitary steps, which cannot wraparound.
05714   // 1*N = -Start; -1*N = Start (mod 2^BW), so:
05715   //   N = Distance (as unsigned)
05716   if (StepC->getValue()->equalsInt(1) || StepC->getValue()->isAllOnesValue()) {
05717     ConstantRange CR = getUnsignedRange(Start);
05718     const SCEV *MaxBECount;
05719     if (!CountDown && CR.getUnsignedMin().isMinValue())
05720       // When counting up, the worst starting value is 1, not 0.
05721       MaxBECount = CR.getUnsignedMax().isMinValue()
05722         ? getConstant(APInt::getMinValue(CR.getBitWidth()))
05723         : getConstant(APInt::getMaxValue(CR.getBitWidth()));
05724     else
05725       MaxBECount = getConstant(CountDown ? CR.getUnsignedMax()
05726                                          : -CR.getUnsignedMin());
05727     return ExitLimit(Distance, MaxBECount, /*MustExit=*/true);
05728   }
05729 
05730   // If the recurrence is known not to wraparound, unsigned divide computes the
05731   // back edge count. (Ideally we would have an "isexact" bit for udiv). We know
05732   // that the value will either become zero (and thus the loop terminates), that
05733   // the loop will terminate through some other exit condition first, or that
05734   // the loop has undefined behavior.  This means we can't "miss" the exit
05735   // value, even with nonunit stride, and exit later via the same branch. Note
05736   // that we can skip this exit if loop later exits via a different
05737   // branch. Hence MustExit=false.
05738   //
05739   // This is only valid for expressions that directly compute the loop exit. It
05740   // is invalid for subexpressions in which the loop may exit through this
05741   // branch even if this subexpression is false. In that case, the trip count
05742   // computed by this udiv could be smaller than the number of well-defined
05743   // iterations.
05744   if (!IsSubExpr && AddRec->getNoWrapFlags(SCEV::FlagNW)) {
05745     const SCEV *Exact =
05746       getUDivExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step);
05747     return ExitLimit(Exact, Exact, /*MustExit=*/false);
05748   }
05749 
05750   // If Step is a power of two that evenly divides Start we know that the loop
05751   // will always terminate.  Start may not be a constant so we just have the
05752   // number of trailing zeros available.  This is safe even in presence of
05753   // overflow as the recurrence will overflow to exactly 0.
05754   const APInt &StepV = StepC->getValue()->getValue();
05755   if (StepV.isPowerOf2() &&
05756       GetMinTrailingZeros(getNegativeSCEV(Start)) >= StepV.countTrailingZeros())
05757     return getUDivExactExpr(Distance, CountDown ? getNegativeSCEV(Step) : Step);
05758 
05759   // Then, try to solve the above equation provided that Start is constant.
05760   if (const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start))
05761     return SolveLinEquationWithOverflow(StepC->getValue()->getValue(),
05762                                         -StartC->getValue()->getValue(),
05763                                         *this);
05764   return getCouldNotCompute();
05765 }
05766 
05767 /// HowFarToNonZero - Return the number of times a backedge checking the
05768 /// specified value for nonzero will execute.  If not computable, return
05769 /// CouldNotCompute
05770 ScalarEvolution::ExitLimit
05771 ScalarEvolution::HowFarToNonZero(const SCEV *V, const Loop *L) {
05772   // Loops that look like: while (X == 0) are very strange indeed.  We don't
05773   // handle them yet except for the trivial case.  This could be expanded in the
05774   // future as needed.
05775 
05776   // If the value is a constant, check to see if it is known to be non-zero
05777   // already.  If so, the backedge will execute zero times.
05778   if (const SCEVConstant *C = dyn_cast<SCEVConstant>(V)) {
05779     if (!C->getValue()->isNullValue())
05780       return getConstant(C->getType(), 0);
05781     return getCouldNotCompute();  // Otherwise it will loop infinitely.
05782   }
05783 
05784   // We could implement others, but I really doubt anyone writes loops like
05785   // this, and if they did, they would already be constant folded.
05786   return getCouldNotCompute();
05787 }
05788 
05789 /// getPredecessorWithUniqueSuccessorForBB - Return a predecessor of BB
05790 /// (which may not be an immediate predecessor) which has exactly one
05791 /// successor from which BB is reachable, or null if no such block is
05792 /// found.
05793 ///
05794 std::pair<BasicBlock *, BasicBlock *>
05795 ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(BasicBlock *BB) {
05796   // If the block has a unique predecessor, then there is no path from the
05797   // predecessor to the block that does not go through the direct edge
05798   // from the predecessor to the block.
05799   if (BasicBlock *Pred = BB->getSinglePredecessor())
05800     return std::make_pair(Pred, BB);
05801 
05802   // A loop's header is defined to be a block that dominates the loop.
05803   // If the header has a unique predecessor outside the loop, it must be
05804   // a block that has exactly one successor that can reach the loop.
05805   if (Loop *L = LI->getLoopFor(BB))
05806     return std::make_pair(L->getLoopPredecessor(), L->getHeader());
05807 
05808   return std::pair<BasicBlock *, BasicBlock *>();
05809 }
05810 
05811 /// HasSameValue - SCEV structural equivalence is usually sufficient for
05812 /// testing whether two expressions are equal, however for the purposes of
05813 /// looking for a condition guarding a loop, it can be useful to be a little
05814 /// more general, since a front-end may have replicated the controlling
05815 /// expression.
05816 ///
05817 static bool HasSameValue(const SCEV *A, const SCEV *B) {
05818   // Quick check to see if they are the same SCEV.
05819   if (A == B) return true;
05820 
05821   // Otherwise, if they're both SCEVUnknown, it's possible that they hold
05822   // two different instructions with the same value. Check for this case.
05823   if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(A))
05824     if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(B))
05825       if (const Instruction *AI = dyn_cast<Instruction>(AU->getValue()))
05826         if (const Instruction *BI = dyn_cast<Instruction>(BU->getValue()))
05827           if (AI->isIdenticalTo(BI) && !AI->mayReadFromMemory())
05828             return true;
05829 
05830   // Otherwise assume they may have a different value.
05831   return false;
05832 }
05833 
05834 /// SimplifyICmpOperands - Simplify LHS and RHS in a comparison with
05835 /// predicate Pred. Return true iff any changes were made.
05836 ///
05837 bool ScalarEvolution::SimplifyICmpOperands(ICmpInst::Predicate &Pred,
05838                                            const SCEV *&LHS, const SCEV *&RHS,
05839                                            unsigned Depth) {
05840   bool Changed = false;
05841 
05842   // If we hit the max recursion limit bail out.
05843   if (Depth >= 3)
05844     return false;
05845 
05846   // Canonicalize a constant to the right side.
05847   if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS)) {
05848     // Check for both operands constant.
05849     if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
05850       if (ConstantExpr::getICmp(Pred,
05851                                 LHSC->getValue(),
05852                                 RHSC->getValue())->isNullValue())
05853         goto trivially_false;
05854       else
05855         goto trivially_true;
05856     }
05857     // Otherwise swap the operands to put the constant on the right.
05858     std::swap(LHS, RHS);
05859     Pred = ICmpInst::getSwappedPredicate(Pred);
05860     Changed = true;
05861   }
05862 
05863   // If we're comparing an addrec with a value which is loop-invariant in the
05864   // addrec's loop, put the addrec on the left. Also make a dominance check,
05865   // as both operands could be addrecs loop-invariant in each other's loop.
05866   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS)) {
05867     const Loop *L = AR->getLoop();
05868     if (isLoopInvariant(LHS, L) && properlyDominates(LHS, L->getHeader())) {
05869       std::swap(LHS, RHS);
05870       Pred = ICmpInst::getSwappedPredicate(Pred);
05871       Changed = true;
05872     }
05873   }
05874 
05875   // If there's a constant operand, canonicalize comparisons with boundary
05876   // cases, and canonicalize *-or-equal comparisons to regular comparisons.
05877   if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(RHS)) {
05878     const APInt &RA = RC->getValue()->getValue();
05879     switch (Pred) {
05880     default: llvm_unreachable("Unexpected ICmpInst::Predicate value!");
05881     case ICmpInst::ICMP_EQ:
05882     case ICmpInst::ICMP_NE:
05883       // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
05884       if (!RA)
05885         if (const SCEVAddExpr *AE = dyn_cast<SCEVAddExpr>(LHS))
05886           if (const SCEVMulExpr *ME = dyn_cast<SCEVMulExpr>(AE->getOperand(0)))
05887             if (AE->getNumOperands() == 2 && ME->getNumOperands() == 2 &&
05888                 ME->getOperand(0)->isAllOnesValue()) {
05889               RHS = AE->getOperand(1);
05890               LHS = ME->getOperand(1);
05891               Changed = true;
05892             }
05893       break;
05894     case ICmpInst::ICMP_UGE:
05895       if ((RA - 1).isMinValue()) {
05896         Pred = ICmpInst::ICMP_NE;
05897         RHS = getConstant(RA - 1);
05898         Changed = true;
05899         break;
05900       }
05901       if (RA.isMaxValue()) {
05902         Pred = ICmpInst::ICMP_EQ;
05903         Changed = true;
05904         break;
05905       }
05906       if (RA.isMinValue()) goto trivially_true;
05907 
05908       Pred = ICmpInst::ICMP_UGT;
05909       RHS = getConstant(RA - 1);
05910       Changed = true;
05911       break;
05912     case ICmpInst::ICMP_ULE:
05913       if ((RA + 1).isMaxValue()) {
05914         Pred = ICmpInst::ICMP_NE;
05915         RHS = getConstant(RA + 1);
05916         Changed = true;
05917         break;
05918       }
05919       if (RA.isMinValue()) {
05920         Pred = ICmpInst::ICMP_EQ;
05921         Changed = true;
05922         break;
05923       }
05924       if (RA.isMaxValue()) goto trivially_true;
05925 
05926       Pred = ICmpInst::ICMP_ULT;
05927       RHS = getConstant(RA + 1);
05928       Changed = true;
05929       break;
05930     case ICmpInst::ICMP_SGE:
05931       if ((RA - 1).isMinSignedValue()) {
05932         Pred = ICmpInst::ICMP_NE;
05933         RHS = getConstant(RA - 1);
05934         Changed = true;
05935         break;
05936       }
05937       if (RA.isMaxSignedValue()) {
05938         Pred = ICmpInst::ICMP_EQ;
05939         Changed = true;
05940         break;
05941       }
05942       if (RA.isMinSignedValue()) goto trivially_true;
05943 
05944       Pred = ICmpInst::ICMP_SGT;
05945       RHS = getConstant(RA - 1);
05946       Changed = true;
05947       break;
05948     case ICmpInst::ICMP_SLE:
05949       if ((RA + 1).isMaxSignedValue()) {
05950         Pred = ICmpInst::ICMP_NE;
05951         RHS = getConstant(RA + 1);
05952         Changed = true;
05953         break;
05954       }
05955       if (RA.isMinSignedValue()) {
05956         Pred = ICmpInst::ICMP_EQ;
05957         Changed = true;
05958         break;
05959       }
05960       if (RA.isMaxSignedValue()) goto trivially_true;
05961 
05962       Pred = ICmpInst::ICMP_SLT;
05963       RHS = getConstant(RA + 1);
05964       Changed = true;
05965       break;
05966     case ICmpInst::ICMP_UGT:
05967       if (RA.isMinValue()) {
05968         Pred = ICmpInst::ICMP_NE;
05969         Changed = true;
05970         break;
05971       }
05972       if ((RA + 1).isMaxValue()) {
05973         Pred = ICmpInst::ICMP_EQ;
05974         RHS = getConstant(RA + 1);
05975         Changed = true;
05976         break;
05977       }
05978       if (RA.isMaxValue()) goto trivially_false;
05979       break;
05980     case ICmpInst::ICMP_ULT:
05981       if (RA.isMaxValue()) {
05982         Pred = ICmpInst::ICMP_NE;
05983         Changed = true;
05984         break;
05985       }
05986       if ((RA - 1).isMinValue()) {
05987         Pred = ICmpInst::ICMP_EQ;
05988         RHS = getConstant(RA - 1);
05989         Changed = true;
05990         break;
05991       }
05992       if (RA.isMinValue()) goto trivially_false;
05993       break;
05994     case ICmpInst::ICMP_SGT:
05995       if (RA.isMinSignedValue()) {
05996         Pred = ICmpInst::ICMP_NE;
05997         Changed = true;
05998         break;
05999       }
06000       if ((RA + 1).isMaxSignedValue()) {
06001         Pred = ICmpInst::ICMP_EQ;
06002         RHS = getConstant(RA + 1);
06003         Changed = true;
06004         break;
06005       }
06006       if (RA.isMaxSignedValue()) goto trivially_false;
06007       break;
06008     case ICmpInst::ICMP_SLT:
06009       if (RA.isMaxSignedValue()) {
06010         Pred = ICmpInst::ICMP_NE;
06011         Changed = true;
06012         break;
06013       }
06014       if ((RA - 1).isMinSignedValue()) {
06015        Pred = ICmpInst::ICMP_EQ;
06016        RHS = getConstant(RA - 1);
06017         Changed = true;
06018        break;
06019       }
06020       if (RA.isMinSignedValue()) goto trivially_false;
06021       break;
06022     }
06023   }
06024 
06025   // Check for obvious equality.
06026   if (HasSameValue(LHS, RHS)) {
06027     if (ICmpInst::isTrueWhenEqual(Pred))
06028       goto trivially_true;
06029     if (ICmpInst::isFalseWhenEqual(Pred))
06030       goto trivially_false;
06031   }
06032 
06033   // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
06034   // adding or subtracting 1 from one of the operands.
06035   switch (Pred) {
06036   case ICmpInst::ICMP_SLE:
06037     if (!getSignedRange(RHS).getSignedMax().isMaxSignedValue()) {
06038       RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
06039                        SCEV::FlagNSW);
06040       Pred = ICmpInst::ICMP_SLT;
06041       Changed = true;
06042     } else if (!getSignedRange(LHS).getSignedMin().isMinSignedValue()) {
06043       LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS,
06044                        SCEV::FlagNSW);
06045       Pred = ICmpInst::ICMP_SLT;
06046       Changed = true;
06047     }
06048     break;
06049   case ICmpInst::ICMP_SGE:
06050     if (!getSignedRange(RHS).getSignedMin().isMinSignedValue()) {
06051       RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS,
06052                        SCEV::FlagNSW);
06053       Pred = ICmpInst::ICMP_SGT;
06054       Changed = true;
06055     } else if (!getSignedRange(LHS).getSignedMax().isMaxSignedValue()) {
06056       LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
06057                        SCEV::FlagNSW);
06058       Pred = ICmpInst::ICMP_SGT;
06059       Changed = true;
06060     }
06061     break;
06062   case ICmpInst::ICMP_ULE:
06063     if (!getUnsignedRange(RHS).getUnsignedMax().isMaxValue()) {
06064       RHS = getAddExpr(getConstant(RHS->getType(), 1, true), RHS,
06065                        SCEV::FlagNUW);
06066       Pred = ICmpInst::ICMP_ULT;
06067       Changed = true;
06068     } else if (!getUnsignedRange(LHS).getUnsignedMin().isMinValue()) {
06069       LHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), LHS,
06070                        SCEV::FlagNUW);
06071       Pred = ICmpInst::ICMP_ULT;
06072       Changed = true;
06073     }
06074     break;
06075   case ICmpInst::ICMP_UGE:
06076     if (!getUnsignedRange(RHS).getUnsignedMin().isMinValue()) {
06077       RHS = getAddExpr(getConstant(RHS->getType(), (uint64_t)-1, true), RHS,
06078                        SCEV::FlagNUW);
06079       Pred = ICmpInst::ICMP_UGT;
06080       Changed = true;
06081     } else if (!getUnsignedRange(LHS).getUnsignedMax().isMaxValue()) {
06082       LHS = getAddExpr(getConstant(RHS->getType(), 1, true), LHS,
06083                        SCEV::FlagNUW);
06084       Pred = ICmpInst::ICMP_UGT;
06085       Changed = true;
06086     }
06087     break;
06088   default:
06089     break;
06090   }
06091 
06092   // TODO: More simplifications are possible here.
06093 
06094   // Recursively simplify until we either hit a recursion limit or nothing
06095   // changes.
06096   if (Changed)
06097     return SimplifyICmpOperands(Pred, LHS, RHS, Depth+1);
06098 
06099   return Changed;
06100 
06101 trivially_true:
06102   // Return 0 == 0.
06103   LHS = RHS = getConstant(ConstantInt::getFalse(getContext()));
06104   Pred = ICmpInst::ICMP_EQ;
06105   return true;
06106 
06107 trivially_false:
06108   // Return 0 != 0.
06109   LHS = RHS = getConstant(ConstantInt::getFalse(getContext()));
06110   Pred = ICmpInst::ICMP_NE;
06111   return true;
06112 }
06113 
06114 bool ScalarEvolution::isKnownNegative(const SCEV *S) {
06115   return getSignedRange(S).getSignedMax().isNegative();
06116 }
06117 
06118 bool ScalarEvolution::isKnownPositive(const SCEV *S) {
06119   return getSignedRange(S).getSignedMin().isStrictlyPositive();
06120 }
06121 
06122 bool ScalarEvolution::isKnownNonNegative(const SCEV *S) {
06123   return !getSignedRange(S).getSignedMin().isNegative();
06124 }
06125 
06126 bool ScalarEvolution::isKnownNonPositive(const SCEV *S) {
06127   return !getSignedRange(S).getSignedMax().isStrictlyPositive();
06128 }
06129 
06130 bool ScalarEvolution::isKnownNonZero(const SCEV *S) {
06131   return isKnownNegative(S) || isKnownPositive(S);
06132 }
06133 
06134 bool ScalarEvolution::isKnownPredicate(ICmpInst::Predicate Pred,
06135                                        const SCEV *LHS, const SCEV *RHS) {
06136   // Canonicalize the inputs first.
06137   (void)SimplifyICmpOperands(Pred, LHS, RHS);
06138 
06139   // If LHS or RHS is an addrec, check to see if the condition is true in
06140   // every iteration of the loop.
06141   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
06142     if (isLoopEntryGuardedByCond(
06143           AR->getLoop(), Pred, AR->getStart(), RHS) &&
06144         isLoopBackedgeGuardedByCond(
06145           AR->getLoop(), Pred, AR->getPostIncExpr(*this), RHS))
06146       return true;
06147   if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(RHS))
06148     if (isLoopEntryGuardedByCond(
06149           AR->getLoop(), Pred, LHS, AR->getStart()) &&
06150         isLoopBackedgeGuardedByCond(
06151           AR->getLoop(), Pred, LHS, AR->getPostIncExpr(*this)))
06152       return true;
06153 
06154   // Otherwise see what can be done with known constant ranges.
06155   return isKnownPredicateWithRanges(Pred, LHS, RHS);
06156 }
06157 
06158 bool
06159 ScalarEvolution::isKnownPredicateWithRanges(ICmpInst::Predicate Pred,
06160                                             const SCEV *LHS, const SCEV *RHS) {
06161   if (HasSameValue(LHS, RHS))
06162     return ICmpInst::isTrueWhenEqual(Pred);
06163 
06164   // This code is split out from isKnownPredicate because it is called from
06165   // within isLoopEntryGuardedByCond.
06166   switch (Pred) {
06167   default:
06168     llvm_unreachable("Unexpected ICmpInst::Predicate value!");
06169   case ICmpInst::ICMP_SGT:
06170     std::swap(LHS, RHS);
06171   case ICmpInst::ICMP_SLT: {
06172     ConstantRange LHSRange = getSignedRange(LHS);
06173     ConstantRange RHSRange = getSignedRange(RHS);
06174     if (LHSRange.getSignedMax().slt(RHSRange.getSignedMin()))
06175       return true;
06176     if (LHSRange.getSignedMin().sge(RHSRange.getSignedMax()))
06177       return false;
06178     break;
06179   }
06180   case ICmpInst::ICMP_SGE:
06181     std::swap(LHS, RHS);
06182   case ICmpInst::ICMP_SLE: {
06183     ConstantRange LHSRange = getSignedRange(LHS);
06184     ConstantRange RHSRange = getSignedRange(RHS);
06185     if (LHSRange.getSignedMax().sle(RHSRange.getSignedMin()))
06186       return true;
06187     if (LHSRange.getSignedMin().sgt(RHSRange.getSignedMax()))
06188       return false;
06189     break;
06190   }
06191   case ICmpInst::ICMP_UGT:
06192     std::swap(LHS, RHS);
06193   case ICmpInst::ICMP_ULT: {
06194     ConstantRange LHSRange = getUnsignedRange(LHS);
06195     ConstantRange RHSRange = getUnsignedRange(RHS);
06196     if (LHSRange.getUnsignedMax().ult(RHSRange.getUnsignedMin()))
06197       return true;
06198     if (LHSRange.getUnsignedMin().uge(RHSRange.getUnsignedMax()))
06199       return false;
06200     break;
06201   }
06202   case ICmpInst::ICMP_UGE:
06203     std::swap(LHS, RHS);
06204   case ICmpInst::ICMP_ULE: {
06205     ConstantRange LHSRange = getUnsignedRange(LHS);
06206     ConstantRange RHSRange = getUnsignedRange(RHS);
06207     if (LHSRange.getUnsignedMax().ule(RHSRange.getUnsignedMin()))
06208       return true;
06209     if (LHSRange.getUnsignedMin().ugt(RHSRange.getUnsignedMax()))
06210       return false;
06211     break;
06212   }
06213   case ICmpInst::ICMP_NE: {
06214     if (getUnsignedRange(LHS).intersectWith(getUnsignedRange(RHS)).isEmptySet())
06215       return true;
06216     if (getSignedRange(LHS).intersectWith(getSignedRange(RHS)).isEmptySet())
06217       return true;
06218 
06219     const SCEV *Diff = getMinusSCEV(LHS, RHS);
06220     if (isKnownNonZero(Diff))
06221       return true;
06222     break;
06223   }
06224   case ICmpInst::ICMP_EQ:
06225     // The check at the top of the function catches the case where
06226     // the values are known to be equal.
06227     break;
06228   }
06229   return false;
06230 }
06231 
06232 /// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
06233 /// protected by a conditional between LHS and RHS.  This is used to
06234 /// to eliminate casts.
06235 bool
06236 ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
06237                                              ICmpInst::Predicate Pred,
06238                                              const SCEV *LHS, const SCEV *RHS) {
06239   // Interpret a null as meaning no loop, where there is obviously no guard
06240   // (interprocedural conditions notwithstanding).
06241   if (!L) return true;
06242 
06243   BasicBlock *Latch = L->getLoopLatch();
06244   if (!Latch)
06245     return false;
06246 
06247   BranchInst *LoopContinuePredicate =
06248     dyn_cast<BranchInst>(Latch->getTerminator());
06249   if (!LoopContinuePredicate ||
06250       LoopContinuePredicate->isUnconditional())
06251     return false;
06252 
06253   return isImpliedCond(Pred, LHS, RHS,
06254                        LoopContinuePredicate->getCondition(),
06255                        LoopContinuePredicate->getSuccessor(0) != L->getHeader());
06256 }
06257 
06258 /// isLoopEntryGuardedByCond - Test whether entry to the loop is protected
06259 /// by a conditional between LHS and RHS.  This is used to help avoid max
06260 /// expressions in loop trip counts, and to eliminate casts.
06261 bool
06262 ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L,
06263                                           ICmpInst::Predicate Pred,
06264                                           const SCEV *LHS, const SCEV *RHS) {
06265   // Interpret a null as meaning no loop, where there is obviously no guard
06266   // (interprocedural conditions notwithstanding).
06267   if (!L) return false;
06268 
06269   // Starting at the loop predecessor, climb up the predecessor chain, as long
06270   // as there are predecessors that can be found that have unique successors
06271   // leading to the original header.
06272   for (std::pair<BasicBlock *, BasicBlock *>
06273          Pair(L->getLoopPredecessor(), L->getHeader());
06274        Pair.first;
06275        Pair = getPredecessorWithUniqueSuccessorForBB(Pair.first)) {
06276 
06277     BranchInst *LoopEntryPredicate =
06278       dyn_cast<BranchInst>(Pair.first->getTerminator());
06279     if (!LoopEntryPredicate ||
06280         LoopEntryPredicate->isUnconditional())
06281       continue;
06282 
06283     if (isImpliedCond(Pred, LHS, RHS,
06284                       LoopEntryPredicate->getCondition(),
06285                       LoopEntryPredicate->getSuccessor(0) != Pair.second))
06286       return true;
06287   }
06288 
06289   return false;
06290 }
06291 
06292 /// RAII wrapper to prevent recursive application of isImpliedCond.
06293 /// ScalarEvolution's PendingLoopPredicates set must be empty unless we are
06294 /// currently evaluating isImpliedCond.
06295 struct MarkPendingLoopPredicate {
06296   Value *Cond;
06297   DenseSet<Value*> &LoopPreds;
06298   bool Pending;
06299 
06300   MarkPendingLoopPredicate(Value *C, DenseSet<Value*> &LP)
06301     : Cond(C), LoopPreds(LP) {
06302     Pending = !LoopPreds.insert(Cond).second;
06303   }
06304   ~MarkPendingLoopPredicate() {
06305     if (!Pending)
06306       LoopPreds.erase(Cond);
06307   }
06308 };
06309 
06310 /// isImpliedCond - Test whether the condition described by Pred, LHS,
06311 /// and RHS is true whenever the given Cond value evaluates to true.
06312 bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred,
06313                                     const SCEV *LHS, const SCEV *RHS,
06314                                     Value *FoundCondValue,
06315                                     bool Inverse) {
06316   MarkPendingLoopPredicate Mark(FoundCondValue, PendingLoopPredicates);
06317   if (Mark.Pending)
06318     return false;
06319 
06320   // Recursively handle And and Or conditions.
06321   if (BinaryOperator *BO = dyn_cast<BinaryOperator>(FoundCondValue)) {
06322     if (BO->getOpcode() == Instruction::And) {
06323       if (!Inverse)
06324         return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) ||
06325                isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse);
06326     } else if (BO->getOpcode() == Instruction::Or) {
06327       if (Inverse)
06328         return isImpliedCond(Pred, LHS, RHS, BO->getOperand(0), Inverse) ||
06329                isImpliedCond(Pred, LHS, RHS, BO->getOperand(1), Inverse);
06330     }
06331   }
06332 
06333   ICmpInst *ICI = dyn_cast<ICmpInst>(FoundCondValue);
06334   if (!ICI) return false;
06335 
06336   // Bail if the ICmp's operands' types are wider than the needed type
06337   // before attempting to call getSCEV on them. This avoids infinite
06338   // recursion, since the analysis of widening casts can require loop
06339   // exit condition information for overflow checking, which would
06340   // lead back here.
06341   if (getTypeSizeInBits(LHS->getType()) <
06342       getTypeSizeInBits(ICI->getOperand(0)->getType()))
06343     return false;
06344 
06345   // Now that we found a conditional branch that dominates the loop or controls
06346   // the loop latch. Check to see if it is the comparison we are looking for.
06347   ICmpInst::Predicate FoundPred;
06348   if (Inverse)
06349     FoundPred = ICI->getInversePredicate();
06350   else
06351     FoundPred = ICI->getPredicate();
06352 
06353   const SCEV *FoundLHS = getSCEV(ICI->getOperand(0));
06354   const SCEV *FoundRHS = getSCEV(ICI->getOperand(1));
06355 
06356   // Balance the types. The case where FoundLHS' type is wider than
06357   // LHS' type is checked for above.
06358   if (getTypeSizeInBits(LHS->getType()) >
06359       getTypeSizeInBits(FoundLHS->getType())) {
06360     if (CmpInst::isSigned(FoundPred)) {
06361       FoundLHS = getSignExtendExpr(FoundLHS, LHS->getType());
06362       FoundRHS = getSignExtendExpr(FoundRHS, LHS->getType());
06363     } else {
06364       FoundLHS = getZeroExtendExpr(FoundLHS, LHS->getType());
06365       FoundRHS = getZeroExtendExpr(FoundRHS, LHS->getType());
06366     }
06367   }
06368 
06369   // Canonicalize the query to match the way instcombine will have
06370   // canonicalized the comparison.
06371   if (SimplifyICmpOperands(Pred, LHS, RHS))
06372     if (LHS == RHS)
06373       return CmpInst::isTrueWhenEqual(Pred);
06374   if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS))
06375     if (FoundLHS == FoundRHS)
06376       return CmpInst::isFalseWhenEqual(FoundPred);
06377 
06378   // Check to see if we can make the LHS or RHS match.
06379   if (LHS == FoundRHS || RHS == FoundLHS) {
06380     if (isa<SCEVConstant>(RHS)) {
06381       std::swap(FoundLHS, FoundRHS);
06382       FoundPred = ICmpInst::getSwappedPredicate(FoundPred);
06383     } else {
06384       std::swap(LHS, RHS);
06385       Pred = ICmpInst::getSwappedPredicate(Pred);
06386     }
06387   }
06388 
06389   // Check whether the found predicate is the same as the desired predicate.
06390   if (FoundPred == Pred)
06391     return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS);
06392 
06393   // Check whether swapping the found predicate makes it the same as the
06394   // desired predicate.
06395   if (ICmpInst::getSwappedPredicate(FoundPred) == Pred) {
06396     if (isa<SCEVConstant>(RHS))
06397       return isImpliedCondOperands(Pred, LHS, RHS, FoundRHS, FoundLHS);
06398     else
06399       return isImpliedCondOperands(ICmpInst::getSwappedPredicate(Pred),
06400                                    RHS, LHS, FoundLHS, FoundRHS);
06401   }
06402 
06403   // Check whether the actual condition is beyond sufficient.
06404   if (FoundPred == ICmpInst::ICMP_EQ)
06405     if (ICmpInst::isTrueWhenEqual(Pred))
06406       if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS))
06407         return true;
06408   if (Pred == ICmpInst::ICMP_NE)
06409     if (!ICmpInst::isTrueWhenEqual(FoundPred))
06410       if (isImpliedCondOperands(FoundPred, LHS, RHS, FoundLHS, FoundRHS))
06411         return true;
06412 
06413   // Otherwise assume the worst.
06414   return false;
06415 }
06416 
06417 /// isImpliedCondOperands - Test whether the condition described by Pred,
06418 /// LHS, and RHS is true whenever the condition described by Pred, FoundLHS,
06419 /// and FoundRHS is true.
06420 bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred,
06421                                             const SCEV *LHS, const SCEV *RHS,
06422                                             const SCEV *FoundLHS,
06423                                             const SCEV *FoundRHS) {
06424   return isImpliedCondOperandsHelper(Pred, LHS, RHS,
06425                                      FoundLHS, FoundRHS) ||
06426          // ~x < ~y --> x > y
06427          isImpliedCondOperandsHelper(Pred, LHS, RHS,
06428                                      getNotSCEV(FoundRHS),
06429                                      getNotSCEV(FoundLHS));
06430 }
06431 
06432 /// isImpliedCondOperandsHelper - Test whether the condition described by
06433 /// Pred, LHS, and RHS is true whenever the condition described by Pred,
06434 /// FoundLHS, and FoundRHS is true.
06435 bool
06436 ScalarEvolution::isImpliedCondOperandsHelper(ICmpInst::Predicate Pred,
06437                                              const SCEV *LHS, const SCEV *RHS,
06438                                              const SCEV *FoundLHS,
06439                                              const SCEV *FoundRHS) {
06440   switch (Pred) {
06441   default: llvm_unreachable("Unexpected ICmpInst::Predicate value!");
06442   case ICmpInst::ICMP_EQ:
06443   case ICmpInst::ICMP_NE:
06444     if (HasSameValue(LHS, FoundLHS) && HasSameValue(RHS, FoundRHS))
06445       return true;
06446     break;
06447   case ICmpInst::ICMP_SLT:
06448   case ICmpInst::ICMP_SLE:
06449     if (isKnownPredicateWithRanges(ICmpInst::ICMP_SLE, LHS, FoundLHS) &&
06450         isKnownPredicateWithRanges(ICmpInst::ICMP_SGE, RHS, FoundRHS))
06451       return true;
06452     break;
06453   case ICmpInst::ICMP_SGT:
06454   case ICmpInst::ICMP_SGE:
06455     if (isKnownPredicateWithRanges(ICmpInst::ICMP_SGE, LHS, FoundLHS) &&
06456         isKnownPredicateWithRanges(ICmpInst::ICMP_SLE, RHS, FoundRHS))
06457       return true;
06458     break;
06459   case ICmpInst::ICMP_ULT:
06460   case ICmpInst::ICMP_ULE:
06461     if (isKnownPredicateWithRanges(ICmpInst::ICMP_ULE, LHS, FoundLHS) &&
06462         isKnownPredicateWithRanges(ICmpInst::ICMP_UGE, RHS, FoundRHS))
06463       return true;
06464     break;
06465   case ICmpInst::ICMP_UGT:
06466   case ICmpInst::ICMP_UGE:
06467     if (isKnownPredicateWithRanges(ICmpInst::ICMP_UGE, LHS, FoundLHS) &&
06468         isKnownPredicateWithRanges(ICmpInst::ICMP_ULE, RHS, FoundRHS))
06469       return true;
06470     break;
06471   }
06472 
06473   return false;
06474 }
06475 
06476 // Verify if an linear IV with positive stride can overflow when in a 
06477 // less-than comparison, knowing the invariant term of the comparison, the 
06478 // stride and the knowledge of NSW/NUW flags on the recurrence.
06479 bool ScalarEvolution::doesIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride,
06480                                          bool IsSigned, bool NoWrap) {
06481   if (NoWrap) return false;
06482 
06483   unsigned BitWidth = getTypeSizeInBits(RHS->getType());
06484   const SCEV *One = getConstant(Stride->getType(), 1);
06485 
06486   if (IsSigned) {
06487     APInt MaxRHS = getSignedRange(RHS).getSignedMax();
06488     APInt MaxValue = APInt::getSignedMaxValue(BitWidth);
06489     APInt MaxStrideMinusOne = getSignedRange(getMinusSCEV(Stride, One))
06490                                 .getSignedMax();
06491 
06492     // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow!
06493     return (MaxValue - MaxStrideMinusOne).slt(MaxRHS);
06494   }
06495 
06496   APInt MaxRHS = getUnsignedRange(RHS).getUnsignedMax();
06497   APInt MaxValue = APInt::getMaxValue(BitWidth);
06498   APInt MaxStrideMinusOne = getUnsignedRange(getMinusSCEV(Stride, One))
06499                               .getUnsignedMax();
06500 
06501   // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow!
06502   return (MaxValue - MaxStrideMinusOne).ult(MaxRHS);
06503 }
06504 
06505 // Verify if an linear IV with negative stride can overflow when in a 
06506 // greater-than comparison, knowing the invariant term of the comparison,
06507 // the stride and the knowledge of NSW/NUW flags on the recurrence.
06508 bool ScalarEvolution::doesIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride,
06509                                          bool IsSigned, bool NoWrap) {
06510   if (NoWrap) return false;
06511 
06512   unsigned BitWidth = getTypeSizeInBits(RHS->getType());
06513   const SCEV *One = getConstant(Stride->getType(), 1);
06514 
06515   if (IsSigned) {
06516     APInt MinRHS = getSignedRange(RHS).getSignedMin();
06517     APInt MinValue = APInt::getSignedMinValue(BitWidth);
06518     APInt MaxStrideMinusOne = getSignedRange(getMinusSCEV(Stride, One))
06519                                .getSignedMax();
06520 
06521     // SMinRHS - SMaxStrideMinusOne < SMinValue => overflow!
06522     return (MinValue + MaxStrideMinusOne).sgt(MinRHS);
06523   }
06524 
06525   APInt MinRHS = getUnsignedRange(RHS).getUnsignedMin();
06526   APInt MinValue = APInt::getMinValue(BitWidth);
06527   APInt MaxStrideMinusOne = getUnsignedRange(getMinusSCEV(Stride, One))
06528                             .getUnsignedMax();
06529 
06530   // UMinRHS - UMaxStrideMinusOne < UMinValue => overflow!
06531   return (MinValue + MaxStrideMinusOne).ugt(MinRHS);
06532 }
06533 
06534 // Compute the backedge taken count knowing the interval difference, the
06535 // stride and presence of the equality in the comparison.
06536 const SCEV *ScalarEvolution::computeBECount(const SCEV *Delta, const SCEV *Step, 
06537                                             bool Equality) {
06538   const SCEV *One = getConstant(Step->getType(), 1);
06539   Delta = Equality ? getAddExpr(Delta, Step)
06540                    : getAddExpr(Delta, getMinusSCEV(Step, One));
06541   return getUDivExpr(Delta, Step);
06542 }
06543 
06544 /// HowManyLessThans - Return the number of times a backedge containing the
06545 /// specified less-than comparison will execute.  If not computable, return
06546 /// CouldNotCompute.
06547 ///
06548 /// @param IsSubExpr is true when the LHS < RHS condition does not directly
06549 /// control the branch. In this case, we can only compute an iteration count for
06550 /// a subexpression that cannot overflow before evaluating true.
06551 ScalarEvolution::ExitLimit
06552 ScalarEvolution::HowManyLessThans(const SCEV *LHS, const SCEV *RHS,
06553                                   const Loop *L, bool IsSigned,
06554                                   bool IsSubExpr) {
06555   // We handle only IV < Invariant
06556   if (!isLoopInvariant(RHS, L))
06557     return getCouldNotCompute();
06558 
06559   const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS);
06560 
06561   // Avoid weird loops
06562   if (!IV || IV->getLoop() != L || !IV->isAffine())
06563     return getCouldNotCompute();
06564 
06565   bool NoWrap = !IsSubExpr &&
06566                 IV->getNoWrapFlags(IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW);
06567 
06568   const SCEV *Stride = IV->getStepRecurrence(*this);
06569 
06570   // Avoid negative or zero stride values
06571   if (!isKnownPositive(Stride))
06572     return getCouldNotCompute();
06573 
06574   // Avoid proven overflow cases: this will ensure that the backedge taken count
06575   // will not generate any unsigned overflow. Relaxed no-overflow conditions
06576   // exploit NoWrapFlags, allowing to optimize in presence of undefined 
06577   // behaviors like the case of C language.
06578   if (!Stride->isOne() && doesIVOverflowOnLT(RHS, Stride, IsSigned, NoWrap))
06579     return getCouldNotCompute();
06580 
06581   ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT
06582                                       : ICmpInst::ICMP_ULT;
06583   const SCEV *Start = IV->getStart();
06584   const SCEV *End = RHS;
06585   if (!isLoopEntryGuardedByCond(L, Cond, getMinusSCEV(Start, Stride), RHS))
06586     End = IsSigned ? getSMaxExpr(RHS, Start)
06587                    : getUMaxExpr(RHS, Start);
06588 
06589   const SCEV *BECount = computeBECount(getMinusSCEV(End, Start), Stride, false);
06590 
06591   APInt MinStart = IsSigned ? getSignedRange(Start).getSignedMin()
06592                             : getUnsignedRange(Start).getUnsignedMin();
06593 
06594   APInt MinStride = IsSigned ? getSignedRange(Stride).getSignedMin()
06595                              : getUnsignedRange(Stride).getUnsignedMin();
06596 
06597   unsigned BitWidth = getTypeSizeInBits(LHS->getType());
06598   APInt Limit = IsSigned ? APInt::getSignedMaxValue(BitWidth) - (MinStride - 1)
06599                          : APInt::getMaxValue(BitWidth) - (MinStride - 1);
06600 
06601   // Although End can be a MAX expression we estimate MaxEnd considering only
06602   // the case End = RHS. This is safe because in the other case (End - Start)
06603   // is zero, leading to a zero maximum backedge taken count.
06604   APInt MaxEnd =
06605     IsSigned ? APIntOps::smin(getSignedRange(RHS).getSignedMax(), Limit)
06606              : APIntOps::umin(getUnsignedRange(RHS).getUnsignedMax(), Limit);
06607 
06608   const SCEV *MaxBECount;
06609   if (isa<SCEVConstant>(BECount))
06610     MaxBECount = BECount;
06611   else
06612     MaxBECount = computeBECount(getConstant(MaxEnd - MinStart),
06613                                 getConstant(MinStride), false);
06614 
06615   if (isa<SCEVCouldNotCompute>(MaxBECount))
06616     MaxBECount = BECount;
06617 
06618   return ExitLimit(BECount, MaxBECount, /*MustExit=*/true);
06619 }
06620 
06621 ScalarEvolution::ExitLimit
06622 ScalarEvolution::HowManyGreaterThans(const SCEV *LHS, const SCEV *RHS,
06623                                      const Loop *L, bool IsSigned,
06624                                      bool IsSubExpr) {
06625   // We handle only IV > Invariant
06626   if (!isLoopInvariant(RHS, L))
06627     return getCouldNotCompute();
06628 
06629   const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(LHS);
06630 
06631   // Avoid weird loops
06632   if (!IV || IV->getLoop() != L || !IV->isAffine())
06633     return getCouldNotCompute();
06634 
06635   bool NoWrap = !IsSubExpr &&
06636                 IV->getNoWrapFlags(IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW);
06637 
06638   const SCEV *Stride = getNegativeSCEV(IV->getStepRecurrence(*this));
06639 
06640   // Avoid negative or zero stride values
06641   if (!isKnownPositive(Stride))
06642     return getCouldNotCompute();
06643 
06644   // Avoid proven overflow cases: this will ensure that the backedge taken count
06645   // will not generate any unsigned overflow. Relaxed no-overflow conditions
06646   // exploit NoWrapFlags, allowing to optimize in presence of undefined 
06647   // behaviors like the case of C language.
06648   if (!Stride->isOne() && doesIVOverflowOnGT(RHS, Stride, IsSigned, NoWrap))
06649     return getCouldNotCompute();
06650 
06651   ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SGT
06652                                       : ICmpInst::ICMP_UGT;
06653 
06654   const SCEV *Start = IV->getStart();
06655   const SCEV *End = RHS;
06656   if (!isLoopEntryGuardedByCond(L, Cond, getAddExpr(Start, Stride), RHS))
06657     End = IsSigned ? getSMinExpr(RHS, Start)
06658                    : getUMinExpr(RHS, Start);
06659 
06660   const SCEV *BECount = computeBECount(getMinusSCEV(Start, End), Stride, false);
06661 
06662   APInt MaxStart = IsSigned ? getSignedRange(Start).getSignedMax()
06663                             : getUnsignedRange(Start).getUnsignedMax();
06664 
06665   APInt MinStride = IsSigned ? getSignedRange(Stride).getSignedMin()
06666                              : getUnsignedRange(Stride).getUnsignedMin();
06667 
06668   unsigned BitWidth = getTypeSizeInBits(LHS->getType());
06669   APInt Limit = IsSigned ? APInt::getSignedMinValue(BitWidth) + (MinStride - 1)
06670                          : APInt::getMinValue(BitWidth) + (MinStride - 1);
06671 
06672   // Although End can be a MIN expression we estimate MinEnd considering only
06673   // the case End = RHS. This is safe because in the other case (Start - End)
06674   // is zero, leading to a zero maximum backedge taken count.
06675   APInt MinEnd =
06676     IsSigned ? APIntOps::smax(getSignedRange(RHS).getSignedMin(), Limit)
06677              : APIntOps::umax(getUnsignedRange(RHS).getUnsignedMin(), Limit);
06678 
06679 
06680   const SCEV *MaxBECount = getCouldNotCompute();
06681   if (isa<SCEVConstant>(BECount))
06682     MaxBECount = BECount;
06683   else
06684     MaxBECount = computeBECount(getConstant(MaxStart - MinEnd), 
06685                                 getConstant(MinStride), false);
06686 
06687   if (isa<SCEVCouldNotCompute>(MaxBECount))
06688     MaxBECount = BECount;
06689 
06690   return ExitLimit(BECount, MaxBECount, /*MustExit=*/true);
06691 }
06692 
06693 /// getNumIterationsInRange - Return the number of iterations of this loop that
06694 /// produce values in the specified constant range.  Another way of looking at
06695 /// this is that it returns the first iteration number where the value is not in
06696 /// the condition, thus computing the exit count. If the iteration count can't
06697 /// be computed, an instance of SCEVCouldNotCompute is returned.
06698 const SCEV *SCEVAddRecExpr::getNumIterationsInRange(ConstantRange Range,
06699                                                     ScalarEvolution &SE) const {
06700   if (Range.isFullSet())  // Infinite loop.
06701     return SE.getCouldNotCompute();
06702 
06703   // If the start is a non-zero constant, shift the range to simplify things.
06704   if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(getStart()))
06705     if (!SC->getValue()->isZero()) {
06706       SmallVector<const SCEV *, 4> Operands(op_begin(), op_end());
06707       Operands[0] = SE.getConstant(SC->getType(), 0);
06708       const SCEV *Shifted = SE.getAddRecExpr(Operands, getLoop(),
06709                                              getNoWrapFlags(FlagNW));
06710       if (const SCEVAddRecExpr *ShiftedAddRec =
06711             dyn_cast<SCEVAddRecExpr>(Shifted))
06712         return ShiftedAddRec->getNumIterationsInRange(
06713                            Range.subtract(SC->getValue()->getValue()), SE);
06714       // This is strange and shouldn't happen.
06715       return SE.getCouldNotCompute();
06716     }
06717 
06718   // The only time we can solve this is when we have all constant indices.
06719   // Otherwise, we cannot determine the overflow conditions.
06720   for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
06721     if (!isa<SCEVConstant>(getOperand(i)))
06722       return SE.getCouldNotCompute();
06723 
06724 
06725   // Okay at this point we know that all elements of the chrec are constants and
06726   // that the start element is zero.
06727 
06728   // First check to see if the range contains zero.  If not, the first
06729   // iteration exits.
06730   unsigned BitWidth = SE.getTypeSizeInBits(getType());
06731   if (!Range.contains(APInt(BitWidth, 0)))
06732     return SE.getConstant(getType(), 0);
06733 
06734   if (isAffine()) {
06735     // If this is an affine expression then we have this situation:
06736     //   Solve {0,+,A} in Range  ===  Ax in Range
06737 
06738     // We know that zero is in the range.  If A is positive then we know that
06739     // the upper value of the range must be the first possible exit value.
06740     // If A is negative then the lower of the range is the last possible loop
06741     // value.  Also note that we already checked for a full range.
06742     APInt One(BitWidth,1);
06743     APInt A     = cast<SCEVConstant>(getOperand(1))->getValue()->getValue();
06744     APInt End = A.sge(One) ? (Range.getUpper() - One) : Range.getLower();
06745 
06746     // The exit value should be (End+A)/A.
06747     APInt ExitVal = (End + A).udiv(A);
06748     ConstantInt *ExitValue = ConstantInt::get(SE.getContext(), ExitVal);
06749 
06750     // Evaluate at the exit value.  If we really did fall out of the valid
06751     // range, then we computed our trip count, otherwise wrap around or other
06752     // things must have happened.
06753     ConstantInt *Val = EvaluateConstantChrecAtConstant(this, ExitValue, SE);
06754     if (Range.contains(Val->getValue()))
06755       return SE.getCouldNotCompute();  // Something strange happened
06756 
06757     // Ensure that the previous value is in the range.  This is a sanity check.
06758     assert(Range.contains(
06759            EvaluateConstantChrecAtConstant(this,
06760            ConstantInt::get(SE.getContext(), ExitVal - One), SE)->getValue()) &&
06761            "Linear scev computation is off in a bad way!");
06762     return SE.getConstant(ExitValue);
06763   } else if (isQuadratic()) {
06764     // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of the
06765     // quadratic equation to solve it.  To do this, we must frame our problem in
06766     // terms of figuring out when zero is crossed, instead of when
06767     // Range.getUpper() is crossed.
06768     SmallVector<const SCEV *, 4> NewOps(op_begin(), op_end());
06769     NewOps[0] = SE.getNegativeSCEV(SE.getConstant(Range.getUpper()));
06770     const SCEV *NewAddRec = SE.getAddRecExpr(NewOps, getLoop(),
06771                                              // getNoWrapFlags(FlagNW)
06772                                              FlagAnyWrap);
06773 
06774     // Next, solve the constructed addrec
06775     std::pair<const SCEV *,const SCEV *> Roots =
06776       SolveQuadraticEquation(cast<SCEVAddRecExpr>(NewAddRec), SE);
06777     const SCEVConstant *R1 = dyn_cast<SCEVConstant>(Roots.first);
06778     const SCEVConstant *R2 = dyn_cast<SCEVConstant>(Roots.second);
06779     if (R1) {
06780       // Pick the smallest positive root value.
06781       if (ConstantInt *CB =
06782           dyn_cast<ConstantInt>(ConstantExpr::getICmp(ICmpInst::ICMP_ULT,
06783                          R1->getValue(), R2->getValue()))) {
06784         if (CB->getZExtValue() == false)
06785           std::swap(R1, R2);   // R1 is the minimum root now.
06786 
06787         // Make sure the root is not off by one.  The returned iteration should
06788         // not be in the range, but the previous one should be.  When solving
06789         // for "X*X < 5", for example, we should not return a root of 2.
06790         ConstantInt *R1Val = EvaluateConstantChrecAtConstant(this,
06791                                                              R1->getValue(),
06792                                                              SE);
06793         if (Range.contains(R1Val->getValue())) {
06794           // The next iteration must be out of the range...
06795           ConstantInt *NextVal =
06796                 ConstantInt::get(SE.getContext(), R1->getValue()->getValue()+1);
06797 
06798           R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE);
06799           if (!Range.contains(R1Val->getValue()))
06800             return SE.getConstant(NextVal);
06801           return SE.getCouldNotCompute();  // Something strange happened
06802         }
06803 
06804         // If R1 was not in the range, then it is a good return value.  Make
06805         // sure that R1-1 WAS in the range though, just in case.
06806         ConstantInt *NextVal =
06807                ConstantInt::get(SE.getContext(), R1->getValue()->getValue()-1);
06808         R1Val = EvaluateConstantChrecAtConstant(this, NextVal, SE);
06809         if (Range.contains(R1Val->getValue()))
06810           return R1;
06811         return SE.getCouldNotCompute();  // Something strange happened
06812       }
06813     }
06814   }
06815 
06816   return SE.getCouldNotCompute();
06817 }
06818 
06819 static const APInt srem(const SCEVConstant *C1, const SCEVConstant *C2) {
06820   APInt A = C1->getValue()->getValue();
06821   APInt B = C2->getValue()->getValue();
06822   uint32_t ABW = A.getBitWidth();
06823   uint32_t BBW = B.getBitWidth();
06824 
06825   if (ABW > BBW)
06826     B = B.sext(ABW);
06827   else if (ABW < BBW)
06828     A = A.sext(BBW);
06829 
06830   return APIntOps::srem(A, B);
06831 }
06832 
06833 static const APInt sdiv(const SCEVConstant *C1, const SCEVConstant *C2) {
06834   APInt A = C1->getValue()->getValue();
06835   APInt B = C2->getValue()->getValue();
06836   uint32_t ABW = A.getBitWidth();
06837   uint32_t BBW = B.getBitWidth();
06838 
06839   if (ABW > BBW)
06840     B = B.sext(ABW);
06841   else if (ABW < BBW)
06842     A = A.sext(BBW);
06843 
06844   return APIntOps::sdiv(A, B);
06845 }
06846 
06847 namespace {
06848 struct SCEVGCD : public SCEVVisitor<SCEVGCD, const SCEV *> {
06849 public:
06850   // Pattern match Step into Start. When Step is a multiply expression, find
06851   // the largest subexpression of Step that appears in Start. When Start is an
06852   // add expression, try to match Step in the subexpressions of Start, non
06853   // matching subexpressions are returned under Remainder.
06854   static const SCEV *findGCD(ScalarEvolution &SE, const SCEV *Start,
06855                              const SCEV *Step, const SCEV **Remainder) {
06856     assert(Remainder && "Remainder should not be NULL");
06857     SCEVGCD R(SE, Step, SE.getConstant(Step->getType(), 0));
06858     const SCEV *Res = R.visit(Start);
06859     *Remainder = R.Remainder;
06860     return Res;
06861   }
06862 
06863   SCEVGCD(ScalarEvolution &S, const SCEV *G, const SCEV *R)
06864       : SE(S), GCD(G), Remainder(R) {
06865     Zero = SE.getConstant(GCD->getType(), 0);
06866     One = SE.getConstant(GCD->getType(), 1);
06867   }
06868 
06869   const SCEV *visitConstant(const SCEVConstant *Constant) {
06870     if (GCD == Constant || Constant == Zero)
06871       return GCD;
06872 
06873     if (const SCEVConstant *CGCD = dyn_cast<SCEVConstant>(GCD)) {
06874       const SCEV *Res = SE.getConstant(gcd(Constant, CGCD));
06875       if (Res != One)
06876         return Res;
06877 
06878       Remainder = SE.getConstant(srem(Constant, CGCD));
06879       Constant = cast<SCEVConstant>(SE.getMinusSCEV(Constant, Remainder));
06880       Res = SE.getConstant(gcd(Constant, CGCD));
06881       return Res;
06882     }
06883 
06884     // When GCD is not a constant, it could be that the GCD is an Add, Mul,
06885     // AddRec, etc., in which case we want to find out how many times the
06886     // Constant divides the GCD: we then return that as the new GCD.
06887     const SCEV *Rem = Zero;
06888     const SCEV *Res = findGCD(SE, GCD, Constant, &Rem);
06889 
06890     if (Res == One || Rem != Zero) {
06891       Remainder = Constant;
06892       return One;
06893     }
06894 
06895     assert(isa<SCEVConstant>(Res) && "Res should be a constant");
06896     Remainder = SE.getConstant(srem(Constant, cast<SCEVConstant>(Res)));
06897     return Res;
06898   }
06899 
06900   const SCEV *visitTruncateExpr(const SCEVTruncateExpr *Expr) {
06901     if (GCD != Expr)
06902       Remainder = Expr;
06903     return GCD;
06904   }
06905 
06906   const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
06907     if (GCD != Expr)
06908       Remainder = Expr;
06909     return GCD;
06910   }
06911 
06912   const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
06913     if (GCD != Expr)
06914       Remainder = Expr;
06915     return GCD;
06916   }
06917 
06918   const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
06919     if (GCD == Expr)
06920       return GCD;
06921 
06922     for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) {
06923       const SCEV *Rem = Zero;
06924       const SCEV *Res = findGCD(SE, Expr->getOperand(e - 1 - i), GCD, &Rem);
06925 
06926       // FIXME: There may be ambiguous situations: for instance,
06927       // GCD(-4 + (3 * %m), 2 * %m) where 2 divides -4 and %m divides (3 * %m).
06928       // The order in which the AddExpr is traversed computes a different GCD
06929       // and Remainder.
06930       if (Res != One)
06931         GCD = Res;
06932       if (Rem != Zero)
06933         Remainder = SE.getAddExpr(Remainder, Rem);
06934     }
06935 
06936     return GCD;
06937   }
06938 
06939   const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
06940     if (GCD == Expr)
06941       return GCD;
06942 
06943     for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) {
06944       if (Expr->getOperand(i) == GCD)
06945         return GCD;
06946     }
06947 
06948     // If we have not returned yet, it means that GCD is not part of Expr.
06949     const SCEV *PartialGCD = One;
06950     for (int i = 0, e = Expr->getNumOperands(); i < e; ++i) {
06951       const SCEV *Rem = Zero;
06952       const SCEV *Res = findGCD(SE, Expr->getOperand(i), GCD, &Rem);
06953       if (Rem != Zero)
06954         // GCD does not divide Expr->getOperand(i).
06955